PHP 批量抓取網頁圖片並本地化存儲：從正則採集到生產級爬蟲封裝

PHP 批量抓取網頁圖片並本地化存儲：從正則採集到生產級爬蟲封裝

手動右鍵保存圖片效率極低。本文演示如何用 PHP 解析網頁 HTML，提取所有 <img> 標籤的 src 屬性，下載並按規則命名存儲，最後封裝為可復用的 ImageHarvester 類，支援斷點續傳、並發控制、去重、異常重試。

基礎版：正則提取 + 串行下載

<?php
function fetchImagesBasic(string $url, string $saveDir): array {
    $html = file_get_contents($url);
    if (!$html) return ['error' => '頁面獲取失敗'];

    // 簡易正則提取 src（生產請用 DOMDocument）
    preg_match_all('/]+src\s*=\s*["\']([^"\']+)["\']/i', $html, $matches);
    $srcList = array_unique($matches[1]);

    if (!is_dir($saveDir)) mkdir($saveDir, 0755, true);

    $results = [];
    foreach ($srcList as $src) {
        $absUrl = resolveUrl($url, $src); // 處理相對路徑
        $ext = parseImageExt($absUrl);
        $filename = md5($absUrl) . '.' . $ext;
        $dest = $saveDir . '/' . $filename;

        $content = @file_get_contents($absUrl);
        if ($content !== false) {
            file_put_contents($dest, $content);
            $results[] = ['url' => $absUrl, 'path' => $dest, 'size' => strlen($content)];
        } else {
            $results[] = ['url' => $absUrl, 'error' => '下載失敗'];
        }
        usleep(200000); // 禮貌延遲 0.2s
    }
    return $results;
}

function resolveUrl(string $base, string $relative): string {
    if (strpos($relative, '://') !== false) return $relative;
    $parsed = parse_url($base);
    $basePath = rtrim($parsed['path'], '/');
    if ($relative[0] === '/') return $parsed['scheme'].'://'.$parsed['host'].$relative;
    return $parsed['scheme'].'://'.$parsed['host'].$basePath.'/'.$relative;
}

function parseImageExt(string $url): string {
    $path = parse_url($url, PHP_URL_PATH);
    $ext = strtolower(pathinfo($path, PATHINFO_EXTENSION));
    return in_array($ext, ['jpg','jpeg','png','gif','webp','bmp','svg']) ? $ext : 'jpg';
}

生產級封裝：ImageHarvester 類

<?php
class ImageHarvester {
    private $saveDir;
    private $maxConcurrency = 5;
    private $timeout = 30;
    private $maxRetries = 2;
    private $userAgent = 'Mozilla/5.0 (compatible; ImageHarvester/1.0)';
    private $db; // SQLite 記錄已下載 URL，支援斷點續傳

    public function __construct(string $saveDir, string $dbPath = ':memory:') {
        $this->saveDir = rtrim($saveDir, '/');
        if (!is_dir($this->saveDir)) mkdir($this->saveDir, 0755, true);
        $this->db = new PDO('sqlite:' . $dbPath);
        $this->db->exec("CREATE TABLE IF NOT EXISTS downloaded (url TEXT PRIMARY KEY, path TEXT, size INT, time INT)");
    }

    public function harvest(string $pageUrl): array {
        $html = $this->fetchPage($pageUrl);
        if (!$html) return ['error' => '頁面獲取失敗'];

        $imgUrls = $this->extractImageUrls($html, $pageUrl);
        $newUrls = $this->filterNewUrls($imgUrls);

        $results = $this->downloadConcurrent($newUrls);
        return ['total' => count($imgUrls), 'new' => count($newUrls), 'results' => $results];
    }

    private function fetchPage(string $url): ?string {
        $ctx = stream_context_create([
            'http' => ['timeout' => $this->timeout, 'user_agent' => $this->userAgent],
            'ssl'  => ['verify_peer' => false, 'verify_peer_name' => false],
        ]);
        return @file_get_contents($url, false, $ctx);
    }

    private function extractImageUrls(string $html, string $baseUrl): array {
        // 推薦用 DOMDocument（容錯性強）
        $dom = new DOMDocument();
        @$dom->loadHTML($html);
        $xpath = new DOMXPath($dom);
        $urls = [];
        foreach ($xpath->query('//img/@src') as $attr) {
            $urls[] = $this->resolveUrl($baseUrl, $attr->nodeValue);
        }
        return array_unique($urls);
    }

    private function filterNewUrls(array $urls): array {
        $placeholders = implode(',', array_fill(0, count($urls), '?'));
        $stmt = $this->db->prepare("SELECT url FROM downloaded WHERE url IN ($placeholders)");
        $stmt->execute($urls);
        $existing = $stmt->fetchAll(PDO::FETCH_COLUMN);
        return array_diff($urls, $existing);
    }

    private function downloadConcurrent(array $urls): array {
        $mh = curl_multi_init();
        $handles = [];
        $results = [];

        // 初始化 easy handles
        foreach ($urls as $url) {
            $ch = curl_init($url);
            curl_setopt_array($ch, [
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_TIMEOUT        => $this->timeout,
                CURLOPT_CONNECTTIMEOUT => 10,
                CURLOPT_FOLLOWLOCATION => true,
                CURLOPT_USERAGENT      => $this->userAgent,
                CURLOPT_SSL_VERIFYPEER => false,
                CURLOPT_PRIVATE        => $url, // 標識 URL
            ]);
            curl_multi_add_handle($mh, $ch);
            $handles[(int)$ch] = $ch;
        }

        // 並發執行循環
        $running = null;
        do {
            curl_multi_exec($mh, $running);
            curl_multi_select($mh, 0.1);
            while ($info = curl_multi_info_read($mh)) {
                $ch = $info['handle'];
                $url = curl_getinfo($ch, CURLINFO_PRIVATE);
                $errno = curl_errno($ch);
                $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);

                if ($errno === 0 && $httpCode === 200) {
                    $content = curl_multi_getcontent($ch);
                    $ext = $this->parseImageExt($url);
                    $filename = md5($url) . '.' . $ext;
                    $dest = $this->saveDir . '/' . $filename;
                    file_put_contents($dest, $content);
                    $size = strlen($content);
                    // 記錄數據庫
                    $stmt = $this->db->prepare("INSERT OR IGNORE INTO downloaded (url, path, size, time) VALUES (?, ?, ?, ?)");
                    $stmt->execute([$url, $dest, $size, time()]);
                    $results[] = ['url' => $url, 'path' => $dest, 'size' => $size];
                } else {
                    $results[] = ['url' => $url, 'error' => "HTTP $httpCode / cURL $errno"];
                }
                curl_multi_remove_handle($mh, $ch);
                curl_close($ch);
            }
        } while ($running);

        curl_multi_close($mh);
        return $results;
    }

    private function resolveUrl(string $base, string $relative): string { /* 同基礎版 */ }
    private function parseImageExt(string $url): string { /* 同基礎版 */ }
}

使用示例

<?php
$harvester = new ImageHarvester('/var/www/images', __DIR__.'/harvest.db');
$result = $harvester->harvest('https://example.com/gallery');
echo "頁面圖片: {$result['total']}, 新增下載: {$result['new']}\n";
foreach ($result['results'] as $r) {
    echo isset($r['path']) ? "✅ {$r['url']} -> {$r['path']} ({$r['size']} bytes)\n"
                           : "❌ {$r['url']}: {$r['error']}\n";
}

關鍵生產特性解析

法律與道德提醒

• ✅ 僅抓取公開授權/CC 協議/自有站點圖片；
• ✅ 遵守 robots.txt 並設置合理延遲；
• ❌ 禁止大規模抓取商業圖庫、受版權保護內容；
• ❌ 禁止繞過防盜鏈、登錄驗證、付費牆獲取非授權資源。