C# 2.0 實作網路蜘蛛 WebSpider（四）：多執行緒下載架構、DownLoadThread 類別、目錄輪換、索引檔案記錄

多執行緒下載策略

• 每條執行緒對應專屬下載目錄
• 目錄檔案數達上限（如 5000）建立新目錄，計數歸零

DownLoadThread 核心程式碼

class DownLoadThread : MyThread
{
    private ParseResource pr = new ParseResource();
    private int currentCount = 0;
    private FileIO fileIO = new FileIO();
    private string path;
    private string[] patterns;
    public bool stop = false;
    public int threadID;

    public DownLoadThread(string[] patterns)
    {
        pr.findUrl += findUrl;
        this.patterns = patterns;
    }

    private void findUrl(string url) => Common.addUrl(url);

    private void openFile()
    {
        fileIO.CloseWriteFile();
        fileIO.OpenWriteFile(path + Common.indexFile);
    }

    public override void run()
    {
        LinkedList urls = new LinkedList();
        path = Common.getDir();
        openFile();

        while (!stop)
        {
            while (!stop && urls.Count == 0)
            {
                Common.getUrls(urls, 20);
                if (urls.Count == 0)
                {
                    Common.threadWait(threadID);
                    sleep(5000);
                }
            }

            StringBuilder sb = new StringBuilder();
            foreach (string url in urls)
            {
                if (stop) break;
                if (currentCount >= Common.maxCount)
                {
                    path = Common.getDir();
                    openFile();
                    currentCount = 0;
                }

                string s = string.Format("{0:D5}", currentCount + 1);
                sb.Remove(0, sb.Length);
                sb.Append(s).Append(":").Append(url);

                try
                {
                    pr.parse(url, path + s, patterns);
                    Common.Count++;
                    fileIO.WriteLine(sb.ToString());
                    currentCount++;
                }
                catch { }
            }
            urls.Clear();
        }
    }
}

關鍵協作類別

• ParseResource：下載與分析 HTML
• FileIO：寫入 index.txt 記錄檔案名稱對應 URL
• Common：靜態類別管理下載隊列、目錄分配、計數器、執行緒等待協調