nsLatin1Prober.cpp

/*  -*- C++ -*-
    SPDX-FileCopyrightText: 1998 Netscape Communications Corporation <developer@mozilla.org>
 
    SPDX-License-Identifier: MIT
*/
 
#include "nsLatin1Prober.h"
#include <stdio.h>
#include <stdlib.h>
 
#define UDF 0 // undefined
#define OTH 1 // other
#define ASC 2 // ascii capital letter
#define ASS 3 // ascii small letter
#define ACV 4 // accent capital vowel
#define ACO 5 // accent capital other
#define ASV 6 // accent small vowel
#define ASO 7 // accent small other
#define CLASS_NUM 8 // total classes
 
namespace kencodingprober
{
static const unsigned char Latin1_CharToClass[] = {
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 00 - 07
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 08 - 0F
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 10 - 17
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 18 - 1F
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 20 - 27
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 28 - 2F
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 30 - 37
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 38 - 3F
    OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, // 40 - 47
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, // 48 - 4F
    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, // 50 - 57
    ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, // 58 - 5F
    OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, // 60 - 67
    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, // 68 - 6F
    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, // 70 - 77
    ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, // 78 - 7F
    OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, // 80 - 87
    OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, // 88 - 8F
    UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // 90 - 97
    OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, // 98 - 9F
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // A0 - A7
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // A8 - AF
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // B0 - B7
    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, // B8 - BF
    ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, // C0 - C7
    ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, // C8 - CF
    ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, // D0 - D7
    ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, // D8 - DF
    ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, // E0 - E7
    ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, // E8 - EF
    ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, // F0 - F7
    ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, // F8 - FF
};
 
/* 0 : illegal
   1 : very unlikely
   2 : normal
   3 : very likely
*/
static const unsigned char Latin1ClassModel[] = {
    /*      UDF OTH ASC ASS ACV ACO ASV ASO  */
    /*UDF*/ 0, 0, 0, 0, 0, 0, 0, 0,
    /*OTH*/ 0, 3, 3, 3, 3, 3, 3, 3,
    /*ASC*/ 0, 3, 3, 3, 3, 3, 3, 3,
    /*ASS*/ 0, 3, 3, 3, 1, 1, 3, 3,
    /*ACV*/ 0, 3, 3, 3, 1, 2, 1, 2,
    /*ACO*/ 0, 3, 3, 3, 3, 3, 3, 3,
    /*ASV*/ 0, 3, 1, 3, 1, 1, 1, 3,
    /*ASO*/ 0, 3, 1, 3, 1, 1, 3, 3,
};
 
void nsLatin1Prober::Reset(void)
{
    mState = eDetecting;
    mLastCharClass = OTH;
    for (int i = 0; i < FREQ_CAT_NUM; i++) {
        mFreqCounter[i] = 0;
    }
}
 
nsProbingState nsLatin1Prober::HandleData(const char *aBuf, unsigned int aLen)
{
    char *newBuf1 = nullptr;
    unsigned int newLen1 = 0;
 
    if (!FilterWithEnglishLetters(aBuf, aLen, &newBuf1, newLen1)) {
        newBuf1 = (char *)aBuf;
        newLen1 = aLen;
    }
 
    for (unsigned int i = 0; i < newLen1; i++) {
        const unsigned char charClass = Latin1_CharToClass[(unsigned char)newBuf1[i]];
        const unsigned char freq = Latin1ClassModel[mLastCharClass * CLASS_NUM + charClass];
        if (freq == 0) {
            mState = eNotMe;
            break;
        }
        mFreqCounter[freq]++;
        mLastCharClass = charClass;
    }
 
    if (newBuf1 != aBuf) {
        free(newBuf1);
    }
 
    return mState;
}
 
float nsLatin1Prober::GetConfidence(void)
{
    if (mState == eNotMe) {
        return 0.01f;
    }
 
    float confidence;
    unsigned int total = 0;
    for (int i = 0; i < FREQ_CAT_NUM; i++) {
        total += mFreqCounter[i];
    }
 
    if (!total) {
        confidence = 0.0f;
    } else {
        confidence = mFreqCounter[3] * 1.0f / total;
        confidence -= mFreqCounter[1] * 20.0f / total;
    }
 
    if (confidence < 0.0f) {
        confidence = 0.0f;
    }
 
    // lower the confidence of latin1 so that other more accurate detector
    // can take priority.
    confidence *= 0.50f;
 
    return confidence;
}
 
#ifdef DEBUG_PROBE
void nsLatin1Prober::DumpStatus()
{
    printf(" Latin1Prober: %1.3f [%s]\r\n", GetConfidence(), GetCharSetName());
}
#endif
}