您现在的位置：程式碼之家 >> 編程語言 > >> C語言 >> C++ >> 關於C++

Unicode/UTF-8/ANSI 編碼自動偵測與轉換類實作 (C++Builder)

編輯：關於C++

前言

處理跨平台、多語言文字檔時，編碼偵測與轉換是常見需求。本文提供一個完整的 C++Builder 類別 ZUtf8_16，支援 ASCII、UTF-8（含無 BOM）、UTF-16 LE/BE 之間的自動偵測與雙向轉換。

支援的編碼類型


enum EncodingType {
    uni8Bit      = 0,  // ASCII/ANSI（預設）
    uni16BE      = 1,  // UTF-16 Big Endian
    uni16LE      = 2,  // UTF-16 Little Endian (Windows Unicode)
    uniUTF8      = 3,  // UTF-8 with BOM (EF BB BF)
    uniUTF8NOBOM = 4   // UTF-8 without BOM
};

類別介面


class ZUtf8_16 {
private:
    EncodingType m_unicodeMode;
    int isUTF8_16(const char* s, unsigned int len, unsigned* cchUnused);
    EncodingType __fastcall DetermineEncoding(unsigned char* data, size_t iLen);

public:
    __fastcall ZUtf8_16();
    __fastcall ~ZUtf8_16();
    EncodingType __fastcall GetEncodingType() { return m_unicodeMode; }
    
    // 從記憶體串流讀取並轉換為 AnsiString
    bool __fastcall LoadFromStream(TMemoryStream* pStream, AnsiString& DestText);
    
    // 將串流內容按指定編碼存檔
    bool __fastcall StreamSaveToFile(TMemoryStream* pStream, 
        AnsiString FileNameA, EncodingType unicodeMode);
};

核心演算法：編碼偵測

BOM 偵測


EncodingType __fastcall ZUtf8_16::DetermineEncoding(unsigned char* data, size_t iLen) {
    m_unicodeMode = uni8Bit;  // 預設 ASCII
    
    if (data[0] == 0xFE && data[1] == 0xFF)       // FE FF
        m_unicodeMode = uni16BE;
    else if (data[0] == 0xFF && data[1] == 0xFE)  // FF FE
        m_unicodeMode = uni16LE;
    else if (data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF)  // EF BB BF
        m_unicodeMode = uniUTF8;
    else if (isUTF8_16(data, iLen, NULL) == 1)    // 無 BOM 的 UTF-8 驗證
        m_unicodeMode = uniUTF8NOBOM;
    
    return m_unicodeMode;
}

無 BOM UTF-8 驗證


int ZUtf8_16::isUTF8_16(const char* s, unsigned int len, unsigned* cchUnused) {
    const unsigned char* sx = (unsigned char*)s;
    const unsigned char* endx = sx + len;
    int ASCII7only = 1;
    
    while (sx < endx) {
        if (!*sx) { ASCII7only = 0; return 0; }  // NUL 視為非 UTF-8
        else if (*sx < 0x80) { sx++; }                    // 0xxxxxxx ASCII
        else if (*sx < 0xC0) { ASCII7only = 0; return 0; } // 10xxxxxx 非法開頭
        else if (*sx < 0xE0) {                            // 110xxxxx 2-byte
            ASCII7only = 0;
            if (sx >= endx-1) break;
            if (!(*sx & 0x1F) || (sx[1] & 0xC0) != 0x80) return 0;
            sx += 2;
        }
        else if (*sx < 0xF0) {                            // 1110xxxx 3-byte
            ASCII7only = 0;
            if (sx >= endx-2) break;
            if (!(*sx & 0x0F) || (sx[1] & 0xC0) != 0x80 || (sx[2] & 0xC0) != 0x80) return 0;
            sx += 3;
        }
        else { return 0; }  // 4+ byte 不支援
    }
    if (cchUnused) *cchUnused = endx - sx;
    return ASCII7only ? 0 : 1;  // 純 ASCII 回傳 0，有效 UTF-8 回傳 1
}

載入與轉換實作


bool __fastcall ZUtf8_16::LoadFromStream(TMemoryStream* pSourceStream, AnsiString& DestText) {
    pSourceStream->Position = 0;
    if (pSourceStream->Size == 0) return true;
    
    m_unicodeMode = DetermineEncoding((unsigned char*)pSourceStream->Memory, pSourceStream->Size);
    pSourceStream->Position = 0;
    
    switch (m_unicodeMode) {
        case uni8Bit:
            // 呼叫端需自行處理 AnsiString 載入
            break;
            
        case uni16BE: {  // UTF-16 BE → 轉 LE 再用 WideCharLenToString
            pSourceStream->Position = 2;  // 跳過 BOM
            int iLen = pSourceStream->Size - 2;
            char* buf = new char[iLen + 2];
            pSourceStream->Read(buf, iLen);
            // 位元組交換：BE → LE
            for (int i = 0; i < iLen; i += 2) {
                char tmp = buf[i]; buf[i] = buf[i+1]; buf[i+1] = tmp;
            }
            DestText = WideCharLenToString((wchar_t*)buf, iLen/2);
            delete[] buf;
            break;
        }
        case uni16LE: {  // UTF-16 LE → 直接讀取
            pSourceStream->Position = 2;
            int iLen = pSourceStream->Size - 2;
            wchar_t* buf = new wchar_t[iLen/2 + 1];
            pSourceStream->Read(buf, iLen);
            DestText = WideString(buf);  // 隱式轉 AnsiString
            delete[] buf;
            break;
        }
        case uniUTF8: {    // UTF-8 with BOM
            pSourceStream->Position = 3;
            // fall through
        }
        case uniUTF8NOBOM: {  // UTF-8 no BOM
            int iLen = pSourceStream->Size - (m_unicodeMode == uniUTF8 ? 3 : 0);
            char* buf = new char[iLen + 3];
            pSourceStream->Read(buf, iLen);
            AnsiString utf8Str = Utf8ToAnsi(buf);  // VCL 內建轉換
            DestText = (utf8Str == "") ? AnsiString((char*)pSourceStream->Memory) : utf8Str;
            delete[] buf;
            break;
        }
    }
    return true;
}

儲存實作


bool __fastcall ZUtf8_16::StreamSaveToFile(TMemoryStream* pStream, 
    AnsiString FileNameA, EncodingType unicodeMode) {
    try {
        pStream->Position = 0;
        switch (unicodeMode) {
            case uni8Bit:
                pStream->SaveToFile(FileNameA);
                break;
            case uni16BE: {
                // ANSI → Unicode → 位元組交換 → 寫入 FE FF BOM
                // 使用 MultiByteToWideChar 轉換
                break;
            }
            case uni16LE: {
                // ANSI → Unicode → 寫入 FF FE BOM
                break;
            }
            case uniUTF8: {
                // ANSI → UTF-8 → 寫入 EF BB BF BOM
                AnsiString utf8 = AnsiToUtf8(source);
                // 寫檔...
                break;
            }
            case uniUTF8NOBOM: {
                // ANSI → UTF-8 → 不寫 BOM
                break;
            }
        }
        return true;
    } catch(...) { return false; }
}

使用範例


bool __fastcall LoadFile(AnsiString strFileName, TStrings* pList) {
    if (!FileExists(strFileName)) {
        MessageBox(0, "檔案不存在", "錯誤", MB_OK|MB_ICONERROR);
        return false;
    }
    
    ZUtf8_16 converter;
    TMemoryStream* stream = new TMemoryStream();
    stream->LoadFromFile(strFileName);
    
    AnsiString text;
    bool ok = converter.LoadFromStream(stream, text);
    
    if (ok) {
        EncodingType enc = converter.GetEncodingType();
        if (enc == uni8Bit)
            pList->LoadFromStream(stream);  // 直接載入
        else
            pList->Text = text;             // 使用轉換後文字
    }
    
    delete stream;
    return ok;
}

關鍵 VCL 函數對照

函數	用途
`WideCharLenToString`	wchar_t* + 長度 → AnsiString
`Utf8ToAnsi`	UTF-8 char* → AnsiString
`AnsiToUtf8`	AnsiString → UTF-8 char*
`MultiByteToWideChar`	Windows API：多位元組 → 寬字元
`WideString`	COM 相容寬字串類別