您现在的位置：程式碼之家 >> 編程語言 > >> 網頁程式設計 >> PHP 程式設計 >> PHP 相關知識

PHP 跨站抓取頁面內容：cURL 實作與常見反爬蟲繞過技巧

編輯：PHP 相關知識

PHP 跨站抓取頁面內容：cURL 實作與常見反爬蟲繞過技巧

需求：從第三方站點（新聞、天氣、產品價格）自動抓取內容供己用。PHP cURL 是最強原生工具，支援 HTTPS、Cookie、偽裝 Header、代理、並發。本文提供基礎抓取代碼、完整封裝類、反爬蟲繞過策略與法律合規提醒。

一、基礎 cURL 抓取（GET）

<?php
function simpleFetch(string $url): ?string {
    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,   // 支援 301/302 重導向
        CURLOPT_MAXREDIRS => 5,
        CURLOPT_TIMEOUT => 20,
        CURLOPT_SSL_VERIFYPEER => false,  // 測試用，生產請配置 CA
        CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    ]);
    $html = curl_exec($ch);
    if (curl_errno($ch)) {
        error_log("cURL Error [$url]: " . curl_error($ch));
        return null;
    }
    curl_close($ch);
    return $html;
}

// 使用
$html = simpleFetch('https://example.com/news');
echo $html;
?>

二、進階：POST 表單、攜帶 Cookie、Session 維持

<?php
class HttpClient {
    private $ch;
    private string $cookieJar;
    public function __construct(string $cookieFile = '') {
        $this->ch = curl_init();
        $this->cookieJar = $cookieFile ?: sys_get_temp_dir() . '/cookiejar_' . spl_object_id($this) . '.txt';
        curl_setopt_array($this->ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS => 10,
            CURLOPT_TIMEOUT => 30,
            CURLOPT_COOKIEJAR => $this->cookieJar,
            CURLOPT_COOKIEFILE => $this->cookieJar,
            CURLOPT_SSL_VERIFYPEER => true,
            CURLOPT_CAINFO => __DIR__ . '/cacert.pem', // 需下載 https://curl.se/ca/cacert.pem
            CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            CURLOPT_ENCODING => 'gzip,deflate', // 自動解壓
        ]);
    }
    public function get(string $url, array $headers = []): ?string {
        curl_setopt($this->ch, CURLOPT_HTTPGET, true);
        curl_setopt($this->ch, CURLOPT_URL, $url);
        if ($headers) curl_setopt($this->ch, CURLOPT_HTTPHEADER, $headers);
        return $this->exec();
    }
    public function post(string $url, array $data, array $headers = []): ?string {
        curl_setopt($this->ch, CURLOPT_POST, true);
        curl_setopt($this->ch, CURLOPT_POSTFIELDS, http_build_query($data));
        curl_setopt($this->ch, CURLOPT_URL, $url);
        if ($headers) curl_setopt($this->ch, CURLOPT_HTTPHEADER, $headers);
        return $this->exec();
    }
    public function postJson(string $url, array $data, array $headers = []): ?string {
        $headers[] = 'Content-Type: application/json';
        curl_setopt($this->ch, CURLOPT_POST, true);
        curl_setopt($this->ch, CURLOPT_POSTFIELDS, json_encode($data));
        curl_setopt($this->ch, CURLOPT_URL, $url);
        curl_setopt($this->ch, CURLOPT_HTTPHEADER, $headers);
        return $this->exec();
    }
    private function exec(): ?string {
        $resp = curl_exec($this->ch);
        if (curl_errno($this->ch)) {
            error_log('cURL Error: ' . curl_error($this->ch));
            return null;
        }
        return $resp;
    }
    public function getInfo(int $opt) { return curl_getinfo($this->ch, $opt); }
    public function __destruct() { curl_close($this->ch); @unlink($this->cookieJar); }
}

// 使用：模擬登入後抓取會員頁
$client = new HttpClient();
$client->post('https://site.com/login', ['user'=>'a','pass'=>'b']);
$html = $client->get('https://site.com/member');
?>

三、常見反爬蟲繞過技巧（僅供防禦研究，請遵守 robots.txt 與服務條款）

反爬手段	繞過/應對策略
User-Agent 檢查	設定真實瀏覽器 UA，定期輪換 UA 池
Referer/Origin 驗證	`CURLOPT_REFERER`、`Origin` 標頭
Cookie/Session 驗證	先請求登入頁獲取 Cookie，後續請求攜帶
IP 限流/封禁	代理 IP 池（住宅代理/數據中心代理）、降低頻率、隨機延遲
JS 渲染/動態加載	改用 Headless Chrome (Puppeteer/PHP-WebDriver) 或分析 XHR/GraphQL 直請 API
字體加密/Canvas 指紋	極難繞過，建議尋找官方 API 或數據源授權
驗證碼/行為分析	接入打碼平台/人工、模擬人類行為軌跡（高成本、低成功率）

四、並發抓取：cURL Multi 或 Swoole/Guzzle

<?php
// cURL Multi 簡易並發（適合幾十個 URL）
function multiFetch(array $urls, int $maxConcurrent = 10): array {
    $mh = curl_multi_init();
    $handles = [];
    $results = [];
    foreach ($urls as $i => $url) {
        $ch = curl_init($url);
        curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>20, CURLOPT_FOLLOWLOCATION=>true]);
        curl_multi_add_handle($mh, $ch);
        $handles[$i] = $ch;
    }
    $running = null;
    do {
        curl_multi_exec($mh, $running);
        curl_multi_select($mh, 0.5);
    } while ($running > 0);
    foreach ($handles as $i => $ch) {
        $results[$i] = ['html' => curl_multi_getcontent($ch), 'info' => curl_getinfo($ch)];
        curl_multi_remove_handle($mh, $ch);
        curl_close($ch);
    }
    curl_multi_cleanup($mh);
    return $results;
}

// 生產建議：使用 Guzzle Promise 或 Swoole 協程，支援數萬並發
?>