lexer.cpp

/*
  This file is part of the KTextTemplate library
 
  SPDX-FileCopyrightText: 2009, 2010, 2011 Stephen Kelly <steveire@gmail.com>
 
  SPDX-License-Identifier: LGPL-2.1-or-later
 
*/
 
#include "lexer_p.h"
 
using namespace KTextTemplate;
 
using TextProcessingState = State<TextProcessingMachine::Type>;
using TextProcessingTransition = TextProcessingMachine::Transition;
 
using ChurningState = LexerObject<TextProcessingState, NullTest, MarksClearer>;
using FinalizeTokenState = LexerObject<TextProcessingState, NullTest, TokenFinalizer>;
using EofHandler = LexerObject<TextProcessingTransition, NullTest, TokenFinalizer>;
using EofHandlerWithTrimming = LexerObject<TextProcessingTransition, NullTest, TokenFinalizerWithTrimming>;
 
using MaybeTemplateSyntaxHandler = CharacterTransition<'{'>;
 
using TagStartHandler = CharacterTransition<'%', MarkStartSyntax>;
using CommentStartHandler = CharacterTransition<'#', MarkStartSyntax>;
using TagEndHandler = CharacterTransition<'%'>;
using CommentEndHandler = CharacterTransition<'#'>;
using BeginValueHandler = CharacterTransition<'{', MarkStartSyntax>;
using MaybeEndValueHandler = CharacterTransition<'}'>;
using NewlineHandler = CharacterTransition<'\n', MarkNewline>;
using EndTemplateSyntaxHandler = CharacterTransition<'}', MarkEndSyntax>;
using NotEndTemplateSyntaxHandler = NegateCharacterTransition<'}'>;
 
using NotBeginTemplateSyntaxHandler = LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'{'>, OrTest<CharacterTest<'#'>, CharacterTest<'%'>>>>>;
 
using NotBeginTemplateSyntaxOrNewlineHandler =
    LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'{'>, OrTest<CharacterTest<'#'>, OrTest<CharacterTest<'%'>, CharacterTest<'\n'>>>>>>;
 
using NotTagCommentOrNewlineHandler =
    LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'#'>, OrTest<CharacterTest<'%'>, CharacterTest<'\n'>>>>>;
 
using NonWhitespaceLineTextHandler = LexerObject<TextProcessingTransition, Negate<OrTest<IsSpace, CharacterTest<'{'>>>>;
 
using WhitespaceNonNewlineHandler = LexerObject<TextProcessingTransition, AndTest<Negate<CharacterTest<'\n'>>, IsSpace>>;
 
using FinalizingLineTextHandler = LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'{'>, IsSpace>>, TokenFinalizer>;
 
using SyntaxBoundaryNewlineHandler = CharacterTransition<'\n', TokenFinalizerWithTrimmingAndNewline>;
using SyntaxBoundaryHandler = CharacterTransition<'{', FinalizeAndMarkStartSyntax>;
 
template<typename Transition>
void addTransition(TextProcessingState *source, Lexer *lexer, TextProcessingState *target)
{
    auto tr = new Transition(lexer, source);
    tr->setTargetState(target);
}
 
TextProcessingMachine *createMachine(Lexer *lexer, Lexer::TrimType type)
{
    auto machine = new TextProcessingMachine;
 
    auto notFinished = new TextProcessingState(machine);
    auto finished = new TextProcessingState(machine);
    machine->setInitialState(notFinished);
 
    auto processingText = new ChurningState(lexer, notFinished);
    auto processingPostNewline = new TextProcessingState(notFinished);
    auto processingBeginTemplateSyntax = new TextProcessingState(notFinished);
    auto processingTag = new TextProcessingState(notFinished);
    auto processingComment = new TextProcessingState(notFinished);
    auto processingValue = new TextProcessingState(notFinished);
    auto maybeProcessingValue = new TextProcessingState(notFinished);
    auto processingEndTag = new TextProcessingState(notFinished);
    auto processingEndComment = new TextProcessingState(notFinished);
    auto processingEndValue = new TextProcessingState(notFinished);
    TextProcessingState *processingPostTemplateSyntax;
 
    if (type == Lexer::SmartTrim)
        processingPostTemplateSyntax = new TextProcessingState(notFinished);
    else
        processingPostTemplateSyntax = new FinalizeTokenState(lexer, notFinished);
    auto processingPostTemplateSyntaxWhitespace = new TextProcessingState(notFinished);
 
    if (type == Lexer::SmartTrim)
        notFinished->setInitialState(processingPostNewline);
    else
        notFinished->setInitialState(processingText);
 
    if (type == Lexer::SmartTrim) {
        addTransition<NewlineHandler>(processingText, lexer, processingPostNewline);
 
        addTransition<NewlineHandler>(processingPostNewline, lexer, processingPostNewline);
        addTransition<MaybeTemplateSyntaxHandler>(processingPostNewline, lexer, processingBeginTemplateSyntax);
        addTransition<NonWhitespaceLineTextHandler>(processingPostNewline, lexer, processingText);
    }
    addTransition<MaybeTemplateSyntaxHandler>(processingText, lexer, processingBeginTemplateSyntax);
 
    addTransition<TagStartHandler>(processingBeginTemplateSyntax, lexer, processingTag);
    addTransition<CommentStartHandler>(processingBeginTemplateSyntax, lexer, processingComment);
    addTransition<BeginValueHandler>(processingBeginTemplateSyntax, lexer, maybeProcessingValue);
 
    if (type == Lexer::SmartTrim) {
        addTransition<NotBeginTemplateSyntaxOrNewlineHandler>(processingBeginTemplateSyntax, lexer, processingText);
        addTransition<NewlineHandler>(processingBeginTemplateSyntax, lexer, processingPostNewline);
    } else {
        addTransition<NotBeginTemplateSyntaxHandler>(processingBeginTemplateSyntax, lexer, processingText);
    }
 
    addTransition<NewlineHandler>(processingTag, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
    addTransition<TagEndHandler>(processingTag, lexer, processingEndTag);
 
    addTransition<NewlineHandler>(processingComment, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
    addTransition<CommentEndHandler>(processingComment, lexer, processingEndComment);
 
    addTransition<TagStartHandler>(maybeProcessingValue, lexer, processingTag);
    addTransition<CommentStartHandler>(maybeProcessingValue, lexer, processingComment);
    addTransition<NotTagCommentOrNewlineHandler>(maybeProcessingValue, lexer, processingValue);
    addTransition<NewlineHandler>(maybeProcessingValue, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
 
    addTransition<NewlineHandler>(processingValue, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
    addTransition<MaybeEndValueHandler>(processingValue, lexer, processingEndValue);
 
    addTransition<NewlineHandler>(processingEndTag, lexer, processingPostNewline);
    addTransition<NotEndTemplateSyntaxHandler>(processingEndTag, lexer, processingTag);
    addTransition<EndTemplateSyntaxHandler>(processingEndTag, lexer, processingPostTemplateSyntax);
 
    addTransition<NewlineHandler>(processingEndComment, lexer, processingPostNewline);
    addTransition<NotEndTemplateSyntaxHandler>(processingEndComment, lexer, processingComment);
    addTransition<EndTemplateSyntaxHandler>(processingEndComment, lexer, processingPostTemplateSyntax);
 
    addTransition<NewlineHandler>(processingEndValue, lexer, processingPostNewline);
    addTransition<NotEndTemplateSyntaxHandler>(processingEndValue, lexer, processingValue);
    addTransition<EndTemplateSyntaxHandler>(processingEndValue, lexer, processingPostTemplateSyntax);
 
    if (type != Lexer::SmartTrim) {
        processingPostTemplateSyntax->setUnconditionalTransition(processingText);
    } else {
        addTransition<SyntaxBoundaryNewlineHandler>(processingPostTemplateSyntax, lexer, processingPostNewline);
        addTransition<WhitespaceNonNewlineHandler>(processingPostTemplateSyntax, lexer, processingPostTemplateSyntaxWhitespace);
        addTransition<FinalizingLineTextHandler>(processingPostTemplateSyntax, lexer, processingText);
        addTransition<SyntaxBoundaryHandler>(processingPostTemplateSyntax, lexer, processingBeginTemplateSyntax);
 
        // NOTE: We only have to transition to this if there was whitespace
        // before the opening tag. Maybe store that in an external state property?
        // Actually, this may be a bug if we try to finalize with trimming and
        // there is no leading whitespace.
        addTransition<SyntaxBoundaryNewlineHandler>(processingPostTemplateSyntaxWhitespace, lexer, processingPostNewline);
        addTransition<FinalizingLineTextHandler>(processingPostTemplateSyntaxWhitespace, lexer, processingText);
        addTransition<SyntaxBoundaryHandler>(processingPostTemplateSyntaxWhitespace, lexer, processingBeginTemplateSyntax);
    }
 
    {
        auto handler = new EofHandler(lexer, notFinished);
        handler->setTargetState(finished);
        notFinished->setEndTransition(handler);
    }
 
    if (type == Lexer::SmartTrim) {
        {
            auto handler = new EofHandlerWithTrimming(lexer, processingPostTemplateSyntaxWhitespace);
            handler->setTargetState(finished);
            processingPostTemplateSyntaxWhitespace->setEndTransition(handler);
        }
        {
            auto handler = new EofHandlerWithTrimming(lexer, processingPostTemplateSyntax);
            handler->setTargetState(finished);
            processingPostTemplateSyntax->setEndTransition(handler);
        }
    }
    return machine;
}
 
Lexer::Lexer(const QString &templateString)
    : m_templateString(templateString)
{
}
 
Lexer::~Lexer() = default;
 
void Lexer::clearMarkers()
{
    m_startSyntaxPosition = -1;
    m_endSyntaxPosition = -1;
    m_newlinePosition = -1;
}
 
void Lexer::reset()
{
    m_tokenList.clear();
    m_lineCount = 0;
    m_upto = 0;
    m_processedUpto = 0;
    clearMarkers();
}
 
QList<Token> Lexer::tokenize(TrimType type)
{
    auto machine = createMachine(this, type);
 
    machine->start();
 
    auto it = m_templateString.constBegin();
    const auto end = m_templateString.constEnd();
 
    reset();
    for (; it != end; ++it, ++m_upto)
        machine->processCharacter(it);
 
    machine->finished();
 
    machine->stop();
 
    delete machine;
 
    return m_tokenList;
}
 
void Lexer::markStartSyntax()
{
    m_startSyntaxPosition = m_upto;
}
 
void Lexer::markEndSyntax()
{
    m_endSyntaxPosition = m_upto + 1;
}
 
void Lexer::markNewline()
{
    m_newlinePosition = m_upto;
    ++m_lineCount;
}
 
void Lexer::finalizeToken()
{
    auto nextPosition = m_upto;
    const auto validSyntax = m_endSyntaxPosition > m_startSyntaxPosition && (m_startSyntaxPosition >= m_processedUpto);
 
    if (validSyntax) {
        Q_ASSERT(m_startSyntaxPosition >= 0);
        nextPosition = m_startSyntaxPosition - 1;
    }
    finalizeToken(nextPosition, validSyntax);
}
 
void Lexer::finalizeTokenWithTrimmedWhitespace()
{
    auto nextPosition = m_upto;
    // We know this to be true because the state machine has already guaranteed
    // it. This method is only called from transition and state actions which
    // occur after valid syntax.
    // TODO Investigate performance and other implications of changing the state
    // machine to assure similar in finalizeToken()
    Q_ASSERT(m_endSyntaxPosition > m_startSyntaxPosition);
 
    Q_ASSERT(m_startSyntaxPosition >= 0);
    if (m_newlinePosition >= 0 && m_newlinePosition >= m_processedUpto)
        nextPosition = qMin(m_startSyntaxPosition - 1, m_newlinePosition);
    else
        nextPosition = m_startSyntaxPosition - 1;
    finalizeToken(nextPosition, true);
}
 
void Lexer::finalizeToken(int nextPosition, bool processSyntax)
{
    {
        Token token;
        token.content = m_templateString.mid(m_processedUpto, nextPosition - m_processedUpto);
        token.tokenType = TextToken;
        token.linenumber = m_lineCount;
        m_tokenList.append(token);
    }
 
    m_processedUpto = nextPosition;
 
    if (!processSyntax)
        return;
 
    m_processedUpto = m_endSyntaxPosition;
 
    const auto differentiator = m_templateString.at(m_startSyntaxPosition);
    if (differentiator == QLatin1Char('#'))
        return;
 
    Token syntaxToken;
    syntaxToken.content = m_templateString.mid(m_startSyntaxPosition + 1, m_endSyntaxPosition - m_startSyntaxPosition - 3).trimmed();
    syntaxToken.linenumber = m_lineCount;
 
    if (differentiator == QLatin1Char('{')) {
        syntaxToken.tokenType = VariableToken;
    } else {
        Q_ASSERT(differentiator == QLatin1Char('%'));
        syntaxToken.tokenType = BlockToken;
    }
    m_tokenList.append(syntaxToken);
}