整合肖波 KTDictSeg 中文分詞器至 Lucene.Net：自訂 Analyzer 實作與效能驗證

Lucene.Net 內建 StandardAnalyzer 對中文分詞僅能單字切分，效果不佳。本文示範如何封裝肖波開源的 KTDictSeg 詞庫分詞器，實作自訂 Analyzer 與 Tokenizer，並實測索引速度較內建分詞器提升 6 倍。

核心實作：KTDictSegAnalyzer

public class KTDictSegAnalyzer : Analyzer
{
    public override TokenStream TokenStream(string fieldName, TextReader reader)
    {
        TokenStream result = new KTDictSegTokenizer(reader);
        result = new LowerCaseFilter(result); // 英文小寫化
        return result;
    }
}

Tokenizer 實作細節

public class KTDictSegTokenizer : Tokenizer
{
    private static CSimpleDictSeg _segmenter;
    private ArrayList _tokenBuffer;
    private int _offset, _position = -1, _length, _start;

    public KTDictSegTokenizer(TextReader input) : base(input)
    {
        if (_segmenter == null)
        {
            _segmenter = new CSimpleDictSeg();
            _segmenter.DictPath = Path.Combine(Environment.CurrentDirectory, "Data") + Path.DirectorySeparatorChar;
            _segmenter.LoadDict();
        }
        _segmenter.FilterStopWords = true;
        _segmenter.MatchName = true;
        _tokenBuffer = _segmenter.Segment(input.ReadToEnd());
    }

    public override Token Next()
    {
        _position++;
        if (_position < _tokenBuffer.Count)
        {
            string term = _tokenBuffer[_position].ToString();
            _length = term.Length;
            _start = _offset;
            _offset += _length;
            return new Token(term, _start, _start + _length);
        }
        return null;
    }
}

建立索引範例

private void BuildIndex()
{
    Analyzer analyzer = new KTDictSegAnalyzer(); // 或 new StandardAnalyzer()
    FSDirectory dir = FSDirectory.GetDirectory(indexPath, true);
    IndexWriter writer = new IndexWriter(dir, analyzer, true);

    foreach (Product p in productDao.GetProducts())
    {
        Document doc = new Document();
        doc.Add(new Field("prodname", p.Name, Field.Store.YES, Field.Index.TOKENIZED));
        doc.Add(new Field("profunction", p.Description ?? "null", Field.Store.YES, Field.Index.UN_TOKENIZED));
        writer.AddDocument(doc);
    }
    writer.Close();
}

效能對比

同樣資料集下，KTDictSegAnalyzer 建立索引耗時約為 StandardAnalyzer 的 1/6。詞庫式分詞準確度顯著優於單字切分，適合中文全文檢索專案。

期待社群提供更優化的整合方案，或分享雨痕 V3 免費版分詞器資源。