C# 2.0 實作網路蜘蛛 WebSpider（九）：UrlFilter 類別實作、URL 正規化、KeyTree 去重、Thread-Safe 鎖定機制

UrlFilter 完整程式碼

class UrlFilter
{
    public static KeyTree urlHashCode = new KeyTree();
    private static object syncUrlHashCode = new object();

    private static string processUrl(string url) // 正規化 URL
    {
        try
        {
            Uri uri = new Uri(url);
            string s = uri.PathAndQuery;
            if (s.Equals("/")) s = "";
            return uri.Host + s;
        }
        catch (Exception e) { throw e; }
    }

    private static bool exists(string url) // 判斷是否存在
    {
        try
        {
            lock (syncUrlHashCode)
            {
                url = processUrl(url);
                return urlHashCode.exists((uint)url.GetHashCode());
            }
        }
        catch (Exception e) { throw e; }
    }

    public static bool isOK(string url) => !exists(url);

    public static void addUrl(string url)
    {
        try
        {
            lock (syncUrlHashCode)
            {
                url = processUrl(url);
                urlHashCode.add((uint)url.GetHashCode());
            }
        }
        catch (Exception e) { throw e; }
    }
}

關鍵設計

• URL 正規化：https://a.com 與 https://a.com/ 視為相同
• KeyTree：自訂雜湊樹結構儲存 URL 雜湊碼，高效去重
• Thread-Safe：lock 保護共享 urlHashCode