C# 多執行緒網頁爬蟲實戰：HttpWebRequest + Regex + WinForms 監控介面

專案目標

• 目標站點：https://tt.mop.com/c44/0/1_{頁碼}.html（共 100 頁）。
• 提取每頁用戶連結 → 進入個人頁 → 解析頭像圖片 URL → 下載儲存。
• 支援**啟動/暫停/繼續/停止**控制，即時顯示進度、速度。

架構設計

1. HttpRequestUtil：封裝 GET 頁面 HTML、下載二進位檔案（支援 User-Agent、Proxy 控制）。
2. Form1：WinForms UI，管理 10 條工作執行緒，每緒處理 10 頁（共 100 頁）。
3. Regex 解析：從 HTML 擷取連結與圖片 URL。
4. 執行緒安全 UI 更新：Control.Invoke 回主執行緒刷新 Label。

核心程式碼

1. HTTP 工具類

public static class HttpRequestUtil
{
    private static readonly string UserAgent = 
        "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)";

    public static string GetPageHtml(string url)
    {
        var req = (HttpWebRequest)WebRequest.Create(url);
        req.UserAgent = UserAgent;
        req.Proxy = null; // 避免公司代理干擾

        using (var resp = (HttpWebResponse)req.GetResponse())
        using (var stream = resp.GetResponseStream())
        using (var reader = new StreamReader(stream, Encoding.UTF8))
            return reader.ReadToEnd();
    }

    public static void HttpDownloadFile(string url)
    {
        int pos = url.LastIndexOf('/') + 1;
        string fileName = url.Substring(pos);
        string dir = Path.Combine(Application.StartupPath, "download");
        Directory.CreateDirectory(dir);
        string filePath = Path.Combine(dir, fileName);
        if (File.Exists(filePath)) return;

        var req = (HttpWebRequest)WebRequest.Create(url);
        req.UserAgent = UserAgent;
        req.Proxy = null;

        using (var resp = (HttpWebResponse)req.GetResponse())
        using (var respStream = resp.GetResponseStream())
        using (var fileStream = new FileStream(filePath, FileMode.Create))
        {
            byte[] buffer = new byte[1024];
            int read;
            while ((read = respStream.Read(buffer, 0, buffer.Length)) > 0)
                fileStream.Write(buffer, 0, read);
        }
    }
}

2. 多執行緒爬取邏輯（Form1 核心）

public partial class Form1 : Form
{
    private readonly List _threads = new List();
    private DateTime _startTime;
    private int _completedPages, _totalPersons, _totalImages;

    private void btnStart_Click(object sender, EventArgs e)
    {
        _startTime = DateTime.Now;
        _completedPages = _totalPersons = _totalImages = 0;
        UpdateUI();

        btnStart.Enabled = false;
        btnPause.Enabled = btnStop.Enabled = true;

        for (int i = 1; i <= 10; i++)
        {
            int threadIndex = i;
            var t = new Thread(() => Worker(threadIndex));
            t.Start();
            _threads.Add(t);
        }
    }

    private void Worker(int threadIndex)
    {
        var regUserLink = new Regex(@"]*?\s+href=""([^""]*?)""");
        var regImg = new Regex(@"
\s* { MessageBox.Show("完成！"); ResetUI(); }));
        }
    }

    // --- UI 更新委派 ---
    private void UpdatePageLabel() =>
        Invoke(new Action(() => lblPage.Text = $"已完成頁數：{_completedPages}"));

    private void UpdatePersonLabel() =>
        Invoke(new Action(() => lblPerson.Text = $"已完成條數：{_totalPersons}"));

    private void UpdateSpeedLabel()
    {
        double secs = (DateTime.Now - _startTime).TotalSeconds;
        double speed = secs > 0 ? _totalImages / secs : 0;
        Invoke(new Action(() => lblSpeed.Text = $"速度：{speed:F1} 張/秒"));
    }

    private void ResetUI()
    {
        btnStart.Enabled = true;
        btnPause.Enabled = btnStop.Enabled = false;
    }

    // --- 暫停/繼續/停止 ---
    private void btnPause_Click(object sender, EventArgs e)
    {
        foreach (var t in _threads)
            if (t.ThreadState == ThreadState.Running) t.Suspend(); // 注意：Suspend 已過時
        btnPause.Enabled = false; btnResume.Enabled = true;
    }

    private void btnResume_Click(object sender, EventArgs e)
    {
        foreach (var t in _threads)
            if (t.ThreadState == ThreadState.Suspended) t.Resume();
        btnPause.Enabled = true; btnResume.Enabled = false;
    }

    private void btnStop_Click(object sender, EventArgs e)
    {
        foreach (var t in _threads) t.Abort(); // 同樣過時，教學用
        ResetUI();
    }

    protected override void OnFormClosing(FormClosingEventArgs e)
    {
        foreach (var t in _threads) t.Abort();
        base.OnFormClosing(e);
    }
}

關鍵技術點解析

現代化重構建議（.NET 6+）

// 使用 HttpClient + HtmlAgilityPack + Task + CancellationToken
var client = new HttpClient { DefaultRequestHeaders = { UserAgent = "..." } };
var cts = new CancellationTokenSource();

var tasks = Enumerable.Range(1, 100)
    .Select(page => DownloadPageAsync(page, client, cts.Token))
    .ToList();

await Task.WhenAll(tasks); // 或用 Parallel.ForEachAsync 限制併發度

async Task DownloadPageAsync(int page, HttpClient http, CancellationToken ct)
{
    string html = await http.GetStringAsync($"https://tt.mop.com/c44/0/1_{page}.html", ct);
    var doc = new HtmlDocument(); doc.LoadHtml(html);
    var userLinks = doc.DocumentNode.SelectNodes("//a[@class='J-userPic']/@href");
    // ... 後續同理，全部 async/await，零阻塞執行緒
}

結語

本範例展示了 **HttpWebRequest + Regex + 手動 Thread 管理** 的傳統爬蟲寫法，適合理解底層運作原理。但實際專案請務必遷移至 **HttpClient + HtmlAgilityPack/AngleSharp + Task/Parallel + CancellationToken** 技術棧，並加入重試、限流、代理池、分散式佇列等機制，構建穩健、可維護、合規的爬蟲系統。