C# 網頁蜘蛛：擷取網頁所有超連結 URL

核心原理

1. 使用 WebClient.DownloadData 取得網頁原始位元組
2. 依網頁編碼解碼為字串 (UTF-8、GB2312 等)
3. 套用正則表達式比對 href 屬性值
4. 輸出或後續處理所有符合的連結

完整實作程式碼

using System;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Collections;

public class LinkExtractor {
    // 支援 http/https、相對路徑、查詢字串的正則
    private static readonly string HrefPattern = 
        @"href\s*=\s*[""']?(?(https?://|/|\./)?[\w\-]+(\.[\w\-]+)*(/[\w\-./?%&=]*)?)[""']?";

    public static string[] ExtractLinks(string url, Encoding encoding = null) {
        using (WebClient client = new WebClient()) {
            // 設定編碼，預設 UTF-8
            if (encoding != null) client.Encoding = encoding;
            
            byte[] data = client.DownloadData(url);
            string html = (encoding ?? Encoding.UTF8).GetString(data);

            MatchCollection matches = Regex.Matches(html, HrefPattern, RegexOptions.IgnoreCase);
            string[] links = new string[matches.Count];
            for (int i = 0; i < matches.Count; i++) {
                links[i] = matches[i].Groups["url"].Value;
            }
            return links;
        }
    }

    // 主程式測試
    public static void Main() {
        string targetUrl = "https://news.163.com";
        try {
            string[] links = ExtractLinks(targetUrl, Encoding.UTF8);
            Console.WriteLine($"共擷取 {links.Length} 個連結：");
            foreach (string link in links) {
                Console.WriteLine(link);
            }
        } catch (Exception ex) {
            Console.WriteLine($"錯誤: {ex.Message}");
        }
    }
}

正則表達式詳細解析

href\s*=\s*["']?          // href =  後可選引號
(?                   // 命名群組 url
    (https?://|/|\./)?    // 協議或相對路徑開頭
    [\w\-]+(\.[\w\-]+)*   // 域名或路徑段
    (/[\w\-./?%&=]*)?     // 後續路徑與查詢字串
)
["']?                     // 結尾可選引號

進階改良建議

現代化非同步版本 (建議生產環境使用)

using System.Net.Http;
using System.Threading.Tasks;

public class AsyncLinkExtractor {
    private static readonly HttpClient _http = new HttpClient();
    private static readonly Regex _hrefRegex = new Regex(HrefPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);

    public static async Task> ExtractAsync(string url) {
        string html = await _http.GetStringAsync(url);
        var matches = _hrefRegex.Matches(html);
        var links = new HashSet(StringComparer.OrdinalIgnoreCase);
        foreach (Match m in matches) {
            string href = m.Groups["url"].Value;
            if (Uri.TryCreate(url, href, out Uri absolute)) {
                links.Add(absolute.AbsoluteUri);
            }
        }
        return links;
    }
}