nsHebrewProber.cpp

Go to the documentation of this file.

/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/*  -*- C++ -*-
*  Copyright (C) 1998 <developer@mozilla.org>
*
*
*  Permission is hereby granted, free of charge, to any person obtaining
*  a copy of this software and associated documentation files (the
*  "Software"), to deal in the Software without restriction, including
*  without limitation the rights to use, copy, modify, merge, publish,
*  distribute, sublicense, and/or sell copies of the Software, and to
*  permit persons to whom the Software is furnished to do so, subject to
*  the following conditions:
*
*  The above copyright notice and this permission notice shall be included 
*  in all copies or substantial portions of the Software.
*
*  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
*  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
*  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
*  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
*  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
*  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
*  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#include "nsHebrewProber.h"
#include <stdio.h>

// windows-1255 / ISO-8859-8 code points of interest
#define FINAL_KAF ('\xea')
#define NORMAL_KAF ('\xeb')
#define FINAL_MEM ('\xed')
#define NORMAL_MEM ('\xee')
#define FINAL_NUN ('\xef')
#define NORMAL_NUN ('\xf0')
#define FINAL_PE ('\xf3')
#define NORMAL_PE ('\xf4')
#define FINAL_TSADI ('\xf5')
#define NORMAL_TSADI ('\xf6')

// Minimum Visual vs Logical final letter score difference.
// If the difference is below this, don't rely solely on the final letter score distance.
#define MIN_FINAL_CHAR_DISTANCE (5)

// Minimum Visual vs Logical model score difference.
// If the difference is below this, don't rely at all on the model score distance.
#define MIN_MODEL_DISTANCE (0.01)

#define VISUAL_HEBREW_NAME ("ISO-8859-8")
#define LOGICAL_HEBREW_NAME ("windows-1255")

namespace kencodingprober {
bool nsHebrewProber::isFinal(char c)
{
  return ((c == FINAL_KAF) || (c == FINAL_MEM) || (c == FINAL_NUN) || (c == FINAL_PE) || (c == FINAL_TSADI));
}

bool nsHebrewProber::isNonFinal(char c)
{
  return ((c == NORMAL_KAF) || (c == NORMAL_MEM) || (c == NORMAL_NUN) || (c == NORMAL_PE));
  // The normal Tsadi is not a good Non-Final letter due to words like 
  // 'lechotet' (to chat) containing an apostrophe after the tsadi. This 
  // apostrophe is converted to a space in FilterWithoutEnglishLetters causing 
  // the Non-Final tsadi to appear at an end of a word even though this is not 
  // the case in the original text.
  // The letters Pe and Kaf rarely display a related behavior of not being a 
  // good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for 
  // example legally end with a Non-Final Pe or Kaf. However, the benefit of 
  // these letters as Non-Final letters outweighs the damage since these words 
  // are quite rare.
}

nsProbingState nsHebrewProber::HandleData(const char* aBuf, unsigned int aLen)
{
  // Both model probers say it's not them. No reason to continue.
  if (GetState() == eNotMe)
    return eNotMe;

  const char *curPtr, *endPtr = aBuf+aLen;
  char cur;

  for (curPtr = (char*)aBuf; curPtr < endPtr; ++curPtr)
  {
    cur = *curPtr;
    if (cur == ' ') // We stand on a space - a word just ended
    {
      if (mBeforePrev != ' ') // *(curPtr-2) was not a space so prev is not a 1 letter word
      {
        if (isFinal(mPrev)) // case (1) [-2:not space][-1:final letter][cur:space]
          ++mFinalCharLogicalScore;
        else if (isNonFinal(mPrev)) // case (2) [-2:not space][-1:Non-Final letter][cur:space]
          ++mFinalCharVisualScore;
      }
    }
    else  // Not standing on a space
    {
      if ((mBeforePrev == ' ') && (isFinal(mPrev)) && (cur != ' ')) // case (3) [-2:space][-1:final letter][cur:not space]
        ++mFinalCharVisualScore;
    }
    mBeforePrev = mPrev;
    mPrev = cur;
  }

  // Forever detecting, till the end or until both model probers return eNotMe (handled above).
  return eDetecting;
}

// Make the decision: is it Logical or Visual?
const char* nsHebrewProber::GetCharSetName()
{
  // If the final letter score distance is dominant enough, rely on it.
  int finalsub = mFinalCharLogicalScore - mFinalCharVisualScore;
  if (finalsub >= MIN_FINAL_CHAR_DISTANCE) 
    return LOGICAL_HEBREW_NAME;
  if (finalsub <= -(MIN_FINAL_CHAR_DISTANCE))
    return VISUAL_HEBREW_NAME;

  // It's not dominant enough, try to rely on the model scores instead.
  float modelsub = mLogicalProb->GetConfidence() - mVisualProb->GetConfidence();
  if (modelsub > MIN_MODEL_DISTANCE)
    return LOGICAL_HEBREW_NAME;
  if (modelsub < -(MIN_MODEL_DISTANCE))
    return VISUAL_HEBREW_NAME;

  // Still no good, back to final letter distance, maybe it'll save the day.
  if (finalsub < 0) 
    return VISUAL_HEBREW_NAME;

  // (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
  return LOGICAL_HEBREW_NAME;
}


void nsHebrewProber::Reset(void)
{
  mFinalCharLogicalScore = 0;
  mFinalCharVisualScore = 0;

  // mPrev and mBeforePrev are initialized to space in order to simulate a word 
  // delimiter at the beginning of the data
  mPrev = ' ';
  mBeforePrev = ' ';
}

nsProbingState nsHebrewProber::GetState(void) 
{
  // Remain active as long as any of the model probers are active.
  if ((mLogicalProb->GetState() == eNotMe) && (mVisualProb->GetState() == eNotMe))
    return eNotMe;
  return eDetecting;
}

#ifdef DEBUG_PROBE
void  nsHebrewProber::DumpStatus()
{
  printf("  HEB: %d - %d [Logical-Visual score]\r\n", mFinalCharLogicalScore, mFinalCharVisualScore);
}
#endif
}