xapiantermgenerator.cpp

/*
 * SPDX-FileCopyrightText: 2014 Vishesh Handa <me@vhanda.in>
 *
 * SPDX-License-Identifier: LGPL-2.1-or-later
 *
 */
 
#include "xapiantermgenerator.h"
 
#include "akonadi_search_xapian_debug.h"
#include <QTextBoundaryFinder>
 
using namespace Akonadi::Search;
 
XapianTermGenerator::XapianTermGenerator(Xapian::Document *doc)
    : m_doc(doc)
{
    if (doc) {
        m_termGen.set_document(*doc);
    }
}
 
void XapianTermGenerator::indexText(const QString &text)
{
    indexText(text, QString());
}
 
void XapianTermGenerator::setDocument(Xapian::Document *doc)
{
    m_doc = doc;
}
 
QStringList XapianTermGenerator::termList(const QString &text)
{
    int start = 0;
    int end = 0;
 
    QStringList list;
    QTextBoundaryFinder bf(QTextBoundaryFinder::Word, text);
    for (; bf.position() != -1; bf.toNextBoundary()) {
        if (bf.boundaryReasons() & QTextBoundaryFinder::StartOfItem) {
            start = bf.position();
            continue;
        } else if (bf.boundaryReasons() & QTextBoundaryFinder::EndOfItem) {
            end = bf.position();
 
            QString str = text.mid(start, end - start);
 
            // Get the string ready for saving
            str = str.toLower();
 
            // Remove all accents
            const QString denormalized = str.normalized(QString::NormalizationForm_KD);
 
            QString cleanString;
            cleanString.reserve(denormalized.size());
            for (const QChar &ch : denormalized) {
                const auto cat = ch.category();
                if (cat != QChar::Mark_NonSpacing && cat != QChar::Mark_SpacingCombining && cat != QChar::Mark_Enclosing) {
                    cleanString.append(ch);
                }
            }
 
            str = cleanString.normalized(QString::NormalizationForm_KC);
            list << str.split(QLatin1Char('_'), Qt::SkipEmptyParts);
        }
    }
 
    return list;
}
 
void XapianTermGenerator::indexText(const QString &text, const QString &prefix, int wdfInc)
{
    const QByteArray par = prefix.toUtf8();
    const QByteArray ta = text.toUtf8();
    m_termGen.index_text(ta.constData(), wdfInc, par.constData());
 
    const QStringList terms = termList(text);
    for (const QString &term : terms) {
        const QByteArray arr = term.toUtf8();
 
        const QByteArray finalArr = par + arr;
        const std::string stdString(finalArr.constData(), finalArr.size());
        m_doc->add_posting(stdString, m_position, wdfInc);
 
        m_position++;
    }
}
 
int XapianTermGenerator::position() const
{
    return m_position;
}
 
void XapianTermGenerator::setPosition(int position)
{
    m_position = position;
}