PHP 多執行緒併發下載：curl_multi 實現高效率批量抓取

單執行緒循環下載效率低，curl_multi 實現真正的並發連線，大幅縮短批量抓取總耗時。本文提供完整封裝與錯誤處理。

核心實現：併發下載器

<?php
/**
 * 併發下載多個 URL
 * @param array $urls URL 陣列
 * @param string $saveDir 儲存目錄
 * @param string $prefix 檔名前綴
 * @param int $maxConcurrent 最大並發數
 * @param int $timeout 單請求逾時秒數
 * @return array 成功儲存的檔案路徑 ['url' => 'local_path', ...]
 */
function downloadConcurrent(array $urls, string $saveDir = './downloads/', string $prefix = '', int $maxConcurrent = 10, int $timeout = 30): array {
    // 過濾有效 URL
    $validUrls = [];
    foreach ($urls as $i => $url) {
        $url = trim($url);
        if ($url !== '' && preg_match('~^https?://~i', $url)) {
            $validUrls[$i] = $url;
        }
    }
    if (empty($validUrls)) return [];
    
    // 建立目錄
    if (!is_dir($saveDir)) {
        mkdir($saveDir, 0755, true);
    }
    
    $handles = [];
    $mh = curl_multi_init();
    $results = [];
    $active = null;
    
    // 初始化易手
    foreach ($validUrls as $idx => $url) {
        $ch = curl_init($url);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS => 5,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_TIMEOUT => $timeout,
            CURLOPT_SSL_VERIFYPEER => true,
            CURLOPT_USERAGENT => 'Mozilla/5.0 (compatible; PHP Concurrent Downloader)',
            CURLOPT_IPRESOLVE => CURL_IPRESOLVE_V4,
            CURLOPT_HEADER => false,
        ]);
        $handles[$idx] = $ch;
        curl_multi_add_handle($mh, $ch);
    }
    
    // 執行併發循環
    do {
        $mrc = curl_multi_exec($mh, $active);
    } while ($mrc === CURLM_CALL_MULTI_PERFORM);
    
    while ($active && $mrc === CURLM_OK) {
        // 等待活動
        if (curl_multi_select($mh) !== -1) {
            do {
                $mrc = curl_multi_exec($mh, $active);
            } while ($mrc === CURLM_CALL_MULTI_PERFORM);
        }
    }
    
    // 收集結果
    foreach ($handles as $idx => $ch) {
        $url = $validUrls[$idx];
        $error = curl_error($ch);
        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        
        if ($error === '' && $httpCode === 200) {
            $content = curl_multi_getcontent($ch);
            if ($content !== false && $content !== '') {
                // 產生檔名
                $ext = pathinfo(parse_url($url, PHP_URL_PATH), PATHINFO_EXTENSION);
                $ext = $ext ?: 'bin';
                $filename = $prefix . ($prefix ? '_' : '') . $idx . '.' . $ext;
                $filepath = $saveDir . $filename;
                
                if (file_put_contents($filepath, $content) !== false) {
                    $results[$url] = $filepath;
                }
            }
        }
        
        curl_multi_remove_handle($mh, $ch);
        curl_close($ch);
    }
    
    curl_multi_close($mh);
    return $results;
}

進階版：控制並發數 (避免連線耗盡)

<?php
class ConcurrentDownloader {
    private $maxConcurrent;
    private $timeout;
    private $saveDir;
    private $results = [];
    private $pending = [];
    private $running = [];
    
    public function __construct(int $maxConcurrent = 10, int $timeout = 30, string $saveDir = './downloads/') {
        $this->maxConcurrent = $maxConcurrent;
        $this->timeout = $timeout;
        $this->saveDir = $saveDir;
        if (!is_dir($saveDir)) mkdir($saveDir, 0755, true);
    }
    
    public function addUrl(string $url, string $filename = ''): self {
        $this->pending[] = ['url' => $url, 'filename' => $filename];
        return $this;
    }
    
    public function addUrls(array $urls): self {
        foreach ($urls as $url) {
            $this->addUrl(is_array($url) ? $url[0] : $url, is_array($url) ? ($url[1] ?? '') : '');
        }
        return $this;
    }
    
    public function execute(): array {
        $mh = curl_multi_init();
        
        while (!empty($this->pending) || !empty($this->running)) {
            // 啟動新請求直到達到並發上限
            while (count($this->running) < $this->maxConcurrent && !empty($this->pending)) {
                $task = array_shift($this->pending);
                $ch = $this->createHandle($task['url']);
                $this->running[(int)$ch] = $task;
                curl_multi_add_handle($mh, $ch);
            }
            
            // 執行
            $active = null;
            do {
                $mrc = curl_multi_exec($mh, $active);
            } while ($mrc === CURLM_CALL_MULTI_PERFORM);
            
            // 等待活動
            if ($active) {
                curl_multi_select($mh, 1.0); // 1秒輪詢
            }
            
            // 檢查完成的請求
            while (($info = curl_multi_info_read($mh)) !== false) {
                $ch = $info['handle'];
                $task = $this->running[(int)$ch] ?? null;
                unset($this->running[(int)$ch]);
                
                if ($task) {
                    $this->processResult($ch, $task);
                }
                
                curl_multi_remove_handle($mh, $ch);
                curl_close($ch);
            }
        }
        
        curl_multi_close($mh);
        return $this->results;
    }
    
    private function createHandle(string $url) {
        $ch = curl_init($url);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS => 5,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_TIMEOUT => $this->timeout,
            CURLOPT_SSL_VERIFYPEER => true,
            CURLOPT_USERAGENT => 'Mozilla/5.0 (compatible; PHP Concurrent Downloader)',
            CURLOPT_IPRESOLVE => CURL_IPRESOLVE_V4,
        ]);
        return $ch;
    }
    
    private function processResult($ch, array $task) {
        $error = curl_error($ch);
        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        $url = $task['url'];
        
        if ($error !== '' || $httpCode !== 200) {
            $this->results[$url] = ['success' => false, 'error' => $error ?: "HTTP $httpCode"];
            return;
        }
        
        $content = curl_multi_getcontent($ch);
        if ($content === false || $content === '') {
            $this->results[$url] = ['success' => false, 'error' => 'Empty content'];
            return;
        }
        
        $filename = $task['filename'] ?: $this->generateFilename($url);
        $filepath = $this->saveDir . $filename;
        
        if (file_put_contents($filepath, $content) !== false) {
            $this->results[$url] = ['success' => true, 'path' => $filepath, 'size' => strlen($content)];
        } else {
            $this->results[$url] = ['success' => false, 'error' => 'Write failed'];
        }
    }
    
    private function generateFilename(string $url): string {
        $path = parse_url($url, PHP_URL_PATH);
        $ext = pathinfo($path, PATHINFO_EXTENSION);
        $ext = $ext ?: 'bin';
        return uniqid('dl_') . '.' . $ext;
    }
}

// 使用
$downloader = new ConcurrentDownloader(20, 30, './images/');
$downloader->addUrls([
    'https://example.com/1.jpg',
    'https://example.com/2.png',
    ['https://example.com/3.gif', 'custom_name.gif'], // 指定檔名
])->execute();

效能比較

重要參數調優

• CURLOPT_CONNECTTIMEOUT：連線建立逾時，建議 5-10 秒
• CURLOPT_TIMEOUT：整個請求逾時，視檔案大小設定
• maxConcurrent：受限於 ulimit -n、頻寬、目標伺服器限制，建議 10-50
• CURLOPT_IPRESOLVE_V4：強制 IPv4，避免 IPv6 連線問題
• curl_multi_select timeout：設定 0.5-1.0 秒平衡 CPU 佔用與回應速度

錯誤處理與重試機制

<?php
function downloadWithRetry(array $urls, int $maxRetries = 3, int $retryDelay = 2): array {
    $results = [];
    $failed = $urls;
    
    for ($attempt = 1; $attempt <= $maxRetries && !empty($failed); $attempt++) {
        $currentResults = downloadConcurrent($failed, './downloads/', "retry{$attempt}_");
        
        $newFailed = [];
        foreach ($failed as $url) {
            if (isset($currentResults[$url])) {
                $results[$url] = $currentResults[$url];
            } else {
                $newFailed[] = $url;
            }
        }
        
        $failed = $newFailed;
        if (!empty($failed)) {
            sleep($retryDelay); // 退避
        }
    }
    
    // 記錄永久失敗
    foreach ($failed as $url) {
        $results[$url] = ['success' => false, 'error' => "Failed after $maxRetries attempts"];
    }
    
    return $results;
}