Java 多頁面廣度優先爬蟲實戰：HttpClient + HtmlParser 完整架構

架構設計：生產者-消費者 + 去重隊列

+----------------+      種子 URL       +------------------+
|  初始化         | ─────────────────► |  Unvisited Queue  | (PriorityQueue 支持優先級/PageRank)
|  (種子頁面)     |                    |  (待訪問)         |
+----------------+                    +------------------+
                                             │
                                    取出 URL (poll)
                                             ▼
                                    +------------------+
                                    |  Downloader      | (HttpClient 4.5+)
                                    |  - 連接/讀取超時  |
                                    | - 重試策略        |
                                    | - User-Agent 池  |
                                    +------------------+
                                             │
                                    保存頁面 / 解析鏈接
                                             ▼
                                    +------------------+
                                    |  Parser          | (HtmlParser / Jsoup)
                                    | - 提取 <a> href  |
                                    | - 提取 <frame> src|
                                    | - 過濾規則        |
                                    +------------------+
                                             │
                                    新鏈接入隊 (addUnvisited)
                                             ▼
                                    +------------------+
                                    |  Visited Set     | (HashSet O(1) 去重)
                                    |  - 防止循環       |
                                    | - 統計已訪問數    |
                                    +------------------+

核心組件代碼

1. 鏈接隊列 (線程安全版)

public class LinkQueue {
    private final Queue<String> unvisited = new PriorityQueue<>();
    private final Set<String> visited = ConcurrentHashMap.newKeySet(); // JDK 8+ 線程安全 Set
    private final ReadWriteLock lock = new ReentrantReadWriteLock();

    public void addUnvisited(String url) {
        if (StringUtils.isBlank(url)) return;
        String normalized = normalizeUrl(url); // 統一協議/大小寫/去片段
        lock.writeLock().lock();
        try {
            if (!visited.contains(normalized) && !unvisited.contains(normalized)) {
                unvisited.offer(normalized);
            }
        } finally {
            lock.writeLock().unlock();
        }
    }

    public String pollUnvisited() {
        lock.writeLock().lock();
        try {
            return unvisited.poll();
        } finally {
            lock.writeLock().unlock();
        }
    }

    public void addVisited(String url) {
        if (StringUtils.isNotBlank(url)) {
            visited.add(normalizeUrl(url));
        }
    }

    public boolean isUnvisitedEmpty() {
        lock.readLock().lock();
        try {
            return unvisited.isEmpty();
        } finally {
            lock.readLock().unlock();
        }
    }

    public int getVisitedCount() {
        return visited.size();
    }

    private String normalizeUrl(String url) {
        try {
            URI uri = new URI(url);
            // 移除 fragment，統一 host 小寫
            return new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), 
                           uri.getQuery(), null).toASCIIString().toLowerCase();
        } catch (URISyntaxException e) {
            return url.toLowerCase();
        }
    }
}

2. 頁面下載器 (HttpClient 最佳配置)

@Slf4j
public class PageDownloader {
    private final CloseableHttpClient httpClient;
    private final RequestConfig requestConfig;

    public PageDownloader() {
        this.requestConfig = RequestConfig.custom()
                .setConnectTimeout(5000)
                .setSocketTimeout(10000)
                .setConnectionRequestTimeout(1000)
                .setRedirectsEnabled(true)
                .setMaxRedirects(3)
                .build();

        this.httpClient = HttpClients.custom()
                .setDefaultRequestConfig(requestConfig)
                .setRetryStrategy(new DefaultHttpRequestRetryStrategy(2, TimeValue.ofSeconds(1)))
                .setUserAgent("Mozilla/5.0 (compatible; JavaCrawler/1.0; +https://example.com/bot)")
                .evictIdleConnections(30, TimeUnit.SECONDS)
                .build();
    }

    public PageDownloadResult download(String url) {
        HttpGet get = new HttpGet(url);
        try (CloseableHttpResponse response = httpClient.execute(get)) {
            int status = response.getStatusLine().getStatusCode();
            if (status != HttpStatus.SC_OK) {
                return PageDownloadResult.failure(url, "HTTP " + status);
            }
            HttpEntity entity = response.getEntity();
            String contentType = EntityUtils.getContentType(entity);
            String charset = ContentType.getOrDefault(entity).getCharset();
            String html = EntityUtils.toString(entity, charset != null ? charset : "UTF-8");
            return PageDownloadResult.success(url, html, contentType);
        } catch (Exception e) {
            log.warn("下載失敗: {} - {}", url, e.getMessage());
            return PageDownloadResult.failure(url, e.getMessage());
        }
    }

    public void close() throws IOException {
        httpClient.close();
    }
}

3. 鏈接解析器 (Jsoup 更現代、健壯)

public class LinkParser {
    private final LinkFilter filter;

    public LinkParser(LinkFilter filter) {
        this.filter = filter;
    }

    public Set<String> extractLinks(String baseUrl, String html) {
        Set<String> links = new HashSet<>();
        Document doc = Jsoup.parse(html, baseUrl);
        
        // 
        for (Element a : doc.select("a[href]")) {
            String href = a.attr("abs:href"); // 自動轉絕對路徑
            if (filter.accept(href)) links.add(href);
        }
        //  /