您现在的位置：程式碼之家 >> 編程語言 > >> 網頁程式設計 >> PHP 程式設計 >> PHP 相關知識

PHP 開發搜尋引擎全棧解析：從爬蟲、索引到排序的完整實作指南

編輯：PHP 相關知識

自建搜尋引擎是理解資訊檢索核心原理的最佳實踐。本文涵蓋網頁爬蟲、內容提取、倒排索引建立、TF-IDF/BM25 排序、分布式擴展等全棧技術，提供可運行的 PHP 核心代碼片段與架構建議。

系統架構總覽

+--------+     +----------+     +------------+     +--------+     +----------+
| 種子 URL | → | 爬蟲調度 | → | 內容解析/去重 | → | 建立索引 | → | 查詢服務 |
+--------+     +----------+     +------------+     +--------+     +----------+
                                      ↓
                              +--------------+
                              | 索引存儲     |
                              | (MySQL/ES)   |
                              +--------------+

一、禮貌爬蟲

<?php
class Crawler {
    private $client;
    private $visited = [];
    private $queue = [];
    private $delayMs = 1000; // 禮貌延遲

    public function __construct() {
        $this->client = new GuzzleHttp\Client([
            'timeout' => 10,
            'headers' => ['User-Agent' => 'MyBot/1.0 (+https://example.com/bot)']
        ]);
    }

    public function crawl(string $seedUrl, int $maxPages = 1000) {
        $this->queue[] = $seedUrl;
        while ($this->queue && count($this->visited) < $maxPages) {
            $url = array_shift($this->queue);
            if (isset($this->visited[$url])) continue;
            $this->visited[$url] = true;

            try {
                $res = $this->client->get($url);
                if ($res->getStatusCode() !== 200) continue;
                $html = (string)$res->getBody();
                $this->parseAndIndex($url, $html);
                $this->extractLinks($html, $url);
            } catch (Exception $e) {
                error_log("Crawl failed: $url - " . $e->getMessage());
            }
            usleep($this->delayMs * 1000);
        }
    }

    private function parseAndIndex(string $url, string $html) {
        // 交給索引器處理
        Indexer::addDocument($url, $html);
    }

    private function extractLinks(string $html, string $baseUrl) {
        $dom = new DOMDocument();
        @$dom->loadHTML($html);
        foreach ($dom->getElementsByTagName('a') as $a) {
            $href = $a->getAttribute('href');
            if ($href) {
                $abs = $this->resolveUrl($href, $baseUrl);
                if ($this->isAllowed($abs) && !isset($this->visited[$abs])) {
                    $this->queue[] = $abs;
                }
            }
        }
    }
    // resolveUrl、isAllowed (robots.txt、域名白名單) 略
}

二、中文分詞與倒排索引

推薦使用 fukuball/jieba-php (結巴分詞 PHP 移植) 或調用外部分詞服務 (SCWS、Elasticsearch IK)。

<?php
class Indexer {
    private static $pdo;
    public static function init(PDO $pdo) { self::$pdo = $pdo; }

    public static function addDocument(string $url, string $html) {
        $text = self::extractText($html);
        $tokens = self::tokenize($text); // 陣列 ['關鍵詞' => 頻率]
        $docId = self::storeDocument($url, $text);
        foreach ($tokens as $term => $tf) {
            self::upsertPosting($term, $docId, $tf);
        }
    }

    private static function tokenize(string $text): array {
        $jieba = new Fukuball\Jieba\Jieba();
        $words = $jieba->cut($text);
        $tf = [];
        foreach ($words as $w) {
            if (mb_strlen($w) > 1) $tf[$w] = ($tf[$w] ?? 0) + 1;
        }
        return $tf;
    }

    private static function upsertPosting(string $term, int $docId, int $tf) {
        $stmt = self::$pdo->prepare("
            INSERT INTO postings (term, doc_id, tf) VALUES (?, ?, ?)
            ON DUPLICATE KEY UPDATE tf = tf + VALUES(tf)
        ");
        $stmt->execute([$term, $docId, $tf]);
    }
    // storeDocument、extractText 略
}

資料表設計

CREATE TABLE documents (
    id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
    url VARCHAR(2048) NOT NULL UNIQUE,
    title VARCHAR(512),
    content LONGTEXT,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
) ENGINE=InnoDB;

CREATE TABLE postings (
    term VARCHAR(100) NOT NULL,
    doc_id BIGINT UNSIGNED NOT NULL,
    tf INT UNSIGNED NOT NULL DEFAULT 1,
    PRIMARY KEY (term, doc_id)
) ENGINE=InnoDB;

三、BM25 排序查詢

<?php
class Searcher {
    private $pdo;
    private $k1 = 1.2, $b = 0.75;
    private $avgdl, $N;

    public function __construct(PDO $pdo) {
        $this->pdo = $pdo;
        $st = $pdo->query("SELECT COUNT(*) FROM documents")->fetchColumn();
        $this->N = (int)$st;
        $this->avgdl = (float)$pdo->query("SELECT AVG(LENGTH(content)) FROM documents")->fetchColumn();
    }

    public function search(string $query, int $limit = 10): array {
        $terms = $this->tokenize($query);
        if (!$terms) return [];

        $placeholders = implode(',', array_fill(0, count($terms), '?'));
        $sql = "
            SELECT d.id, d.url, d.title,
                   SUM(
                       LOG(({$this->N} - df + 0.5) / (df + 0.5)) *
                       (tf * ({$this->k1} + 1)) /
                       (tf + {$this->k1} * (1 - {$this->b} + {$this->b} * dl / {$this->avgdl}))
                   ) AS score
            FROM documents d
            JOIN postings p ON d.id = p.doc_id
            JOIN (
                SELECT term, COUNT(*) AS df FROM postings WHERE term IN ($placeholders) GROUP BY term
            ) t ON p.term = t.term
            WHERE p.term IN ($placeholders)
            GROUP BY d.id
            ORDER BY score DESC
            LIMIT $limit
        ";
        $params = array_merge(array_keys($terms), array_keys($terms));
        $stmt = $this->pdo->prepare($sql);
        $stmt->execute($params);
        return $stmt->fetchAll(PDO::FETCH_ASSOC);
    }
    // tokenize 同 Indexer
}

四、工程化演進路線

階段	技術選型	關鍵指標
單機原型	PHP + MySQL + Jieba	百萬文檔、秒級查詢
垂直擴展	Elasticsearch (IK 分詞)	千萬文檔、毫秒級查詢、高亮/建議
水平擴展	ES 集群 + Kafka 爬蟲解耦 + Redis 快取	億級文檔、高可用、近實時索引