C# 下載網頁原始碼：WebClient、HttpClient 完整範例與編碼處理

概述

抓取網頁 HTML 是爬蟲、整合舊系統、SEO 分析的基礎操作。本文比較 WebClient（舊版）、HttpClient（現代標準）兩種方式，重點解決編碼自動偵測、壓縮支援、逾時與錯誤處理。

方案一：HttpClient（.NET Core / .NET 5+ 推薦）

using System;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;

public class WebScraper
{
    private static readonly HttpClient _client = new HttpClient(new HttpClientHandler
    {
        AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate,
        UseCookies = false,
        AllowAutoRedirect = true,
        MaxAutomaticRedirections = 5
    })
    {
        Timeout = TimeSpan.FromSeconds(30),
        DefaultRequestHeaders = 
        {
            { "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" },
            { "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" },
            { "Accept-Language", "zh-TW,zh;q=0.9,en;q=0.8" }
        }
    };

    public static async Task DownloadHtmlAsync(string url)
    {
        try
        {
            using (HttpResponseMessage response = await _client.GetAsync(url))
            {
                response.EnsureSuccessStatusCode();
                
                // 關鍵：以位元組讀取，再自動偵測編碼
                byte[] contentBytes = await response.Content.ReadAsByteArrayAsync();
                return DetectAndDecode(contentBytes, response.Content.Headers.ContentType?.CharSet);
            }
        }
        catch (HttpRequestException ex)
        {
            throw new Exception($"HTTP 錯誤: {ex.Message}", ex);
        }
        catch (TaskCanceledException)
        {
            throw new TimeoutException("請求逾時");
        }
    }

    // 編碼偵測邏輯
    private static string DetectAndDecode(byte[] bytes, string charsetHint)
    {
        // 1. 優先使用 HTTP 標頭指定編碼
        if (!string.IsNullOrEmpty(charsetHint))
        {
            try { return Encoding.GetEncoding(charsetHint).GetString(bytes); }
            catch { /* 忽略，嘗試下一步 */ }
        }

        // 2. 掃描 HTML meta 標籤 charset
        string raw = Encoding.ASCII.GetString(bytes.Take(Math.Min(bytes.Length, 2048)).ToArray());
        var match = System.Text.RegularExpressions.Regex.Match(raw, @"]+charset\s*=\s*[""']?([\w-]+)", 
            System.Text.RegularExpressions.RegexOptions.IgnoreCase);
        if (match.Success)
        {
            try { return Encoding.GetEncoding(match.Groups[1].Value).GetString(bytes); }
            catch { /* 忽略 */ }
        }

        // 3. 使用 Ude.NetStandard (NuGet) 自動偵測
        // return CharsetDetector.Detect(bytes); // 需安裝套件

        // 4. 回退：UTF-8 → Big5 → GB2312
        foreach (string enc in new[] { "utf-8", "big5", "gb2312" })
        {
            try { return Encoding.GetEncoding(enc).GetString(bytes); }
            catch { }
        }

        return Encoding.UTF8.GetString(bytes); // 最後嘗試
    }
}

方案二：WebClient（.NET Framework 維護專案）

using System;
using System.Net;
using System.Text;

public class LegacyScraper
{
    public string DownloadHtml(string url)
    {
        using (WebClient client = new WebClient())
        {
            client.Headers.Add("User-Agent", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1)");
            client.Encoding = Encoding.UTF8; // 預設嘗試 UTF-8
            
            try
            {
                byte[] data = client.DownloadData(url);
                return DetectEncoding(data, client.ResponseHeaders["Content-Type"]);
            }
            catch (WebException ex)
            {
                throw new Exception($"下載失敗: {ex.Message}", ex);
            }
        }
    }

    private string DetectEncoding(byte[] data, string contentType)
    {
        // 同 HttpClient 版本的 DetectAndDecode 邏輯
        // ...
        return Encoding.GetEncoding("GB2312").GetString(data); // 原文範例預設 GB2312
    }
}

進階功能擴充

支援 POST 表單提交

public static async Task PostFormAsync(string url, Dictionary formData)
{
    using (var content = new FormUrlEncodedContent(formData))
    using (var response = await _client.PostAsync(url, content))
    {
        response.EnsureSuccessStatusCode();
        byte[] bytes = await response.Content.ReadAsByteArrayAsync();
        return DetectAndDecode(bytes, response.Content.Headers.ContentType?.CharSet);
    }
}

處理 Cookie 會話（登入後抓取）

var handler = new HttpClientHandler
{
    CookieContainer = new CookieContainer(),
    UseCookies = true
};
using (var client = new HttpClient(handler))
{
    // 1. 登入取得 Cookie
    await client.PostAsync(loginUrl, new FormUrlEncodedContent(credentials));
    
    // 2. 後續請求自動攜帶 Cookie
    string html = await client.GetStringAsync(protectedUrl);
}

串流處理大型回應（避免記憶體壓力）

public static async Task SaveHtmlToFileAsync(string url, string filePath)
{
    using (var response = await _client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead))
    using (var stream = await response.Content.ReadAsStreamAsync())
    using (var fileStream = File.Create(filePath))
    {
        await stream.CopyToAsync(fileStream);
    }
}

編碼偵測最佳實踐

常見問題排查

• 亂碼：編碼偵測失敗 → 加強 meta 掃描、嘗試常見編碼列表、引入 Ude 庫
• 403 Forbidden：缺少 User-Agent、Referer、Cookie → 模擬瀏覽器標頭
• 壓縮內容解壓失敗：啟用 AutomaticDecompression 或手動 GZipStream
• HTTPS 憑證錯誤：開發環境可暫時忽略 ServerCertificateCustomValidationCallback（生產嚴禁）
• 連線洩漏：HttpClient 單例模式、using 釋放回應、設定 PooledConnectionLifetime

現代化建議

• 安裝 HtmlAgilityPack 或 AngleSharp 解析 HTML，勿用 Regex
• 使用 Polly 實作重試、斷路器、逾時策略
• 遵守 robots.txt、設定合理延遲、識別爬蟲身分
• 考慮 HttpClientFactory (DI) 管理生命週期

總結

抓取網頁核心在於：正確的 HTTP 標頭、自動解壓、智慧編碼偵測、錯誤重試。現代專案請使用 HttpClient 單例 + HtmlAgilityPack，避開 WebClient 過時 API。編碼偵測無銀彈，需針對目標網站調整策略。