9 #include <QCoreApplication>
13 #include <QMutableMapIterator>
14 #include <QRegularExpression>
16 #include <QXmlStreamReader>
18 #ifdef QT_XMLPATTERNS_LIB
20 #include <QXmlSchemaValidator>
23 #include "../lib/worddelimiters_p.h"
24 #include "../lib/xml_p.h"
28 using KSyntaxHighlighting::WordDelimiters;
29 using KSyntaxHighlighting::Xml::attrToBool;
35 void setDefinition(
const T &verStr,
const QString &filename,
const QString &name)
37 m_currentDefinition = &*m_definitions.insert(name, Definition{});
38 m_currentDefinition->languageName =
name;
39 m_currentDefinition->filename = filename;
40 m_currentDefinition->kateVersionStr = verStr.toString();
41 m_currentKeywords =
nullptr;
42 m_currentContext =
nullptr;
46 qWarning() << filename <<
"invalid kateversion" << verStr;
49 m_currentDefinition->kateVersion = {verStr.
left(idx).
toInt(), verStr.mid(idx + 1).toInt()};
56 if (m_currentContext) {
57 m_currentContext->rules.push_back(Context::Rule{});
58 auto &
rule = m_currentContext->rules.back();
59 m_success =
rule.parseElement(m_currentDefinition->filename, xml) && m_success;
60 m_currentContext->hasDynamicRule = m_currentContext->hasDynamicRule ||
rule.dynamic == XmlBool::True;
61 }
else if (m_currentKeywords) {
62 m_success = m_currentKeywords->items.parseElement(m_currentDefinition->filename, xml) && m_success;
63 }
else if (xml.
name() == QStringLiteral(
"context")) {
64 processContextElement(xml);
65 }
else if (xml.
name() == QStringLiteral(
"list")) {
66 processListElement(xml);
67 }
else if (xml.
name() == QStringLiteral(
"keywords")) {
68 m_success = m_currentDefinition->parseKeywords(xml) && m_success;
69 }
else if (xml.
name() == QStringLiteral(
"emptyLine")) {
70 m_success = parseEmptyLine(m_currentDefinition->filename, xml) && m_success;
71 }
else if (xml.
name() == QStringLiteral(
"itemData")) {
72 m_success = m_currentDefinition->itemDatas.parseElement(m_currentDefinition->filename, xml) && m_success;
75 if (m_currentContext && xml.
name() == QStringLiteral(
"context")) {
76 m_currentContext =
nullptr;
77 }
else if (m_currentKeywords && xml.
name() == QStringLiteral(
"list")) {
78 m_currentKeywords =
nullptr;
84 void resolveContexts()
87 while (def.hasNext()) {
89 auto &definition = def.value();
90 auto &contexts = definition.contexts;
92 if (contexts.isEmpty()) {
93 qWarning() << definition.filename <<
"has no context";
98 auto markAsUsedContext = [](ContextName &contextName) {
99 if (!contextName.stay && contextName.context) {
100 contextName.context->isOnlyIncluded =
false;
105 while (contextIt.hasNext()) {
107 auto &context = contextIt.value();
108 resolveContextName(definition, context, context.lineEndContext, context.line);
109 resolveContextName(definition, context, context.lineEmptyContext, context.line);
110 resolveContextName(definition, context, context.fallthroughContext, context.line);
111 markAsUsedContext(context.lineEndContext);
112 markAsUsedContext(context.lineEmptyContext);
113 markAsUsedContext(context.fallthroughContext);
114 for (
auto &
rule : context.rules) {
115 rule.parentContext = &context;
116 resolveContextName(definition, context,
rule.context,
rule.line);
117 if (
rule.type != Context::Rule::Type::IncludeRules) {
118 markAsUsedContext(
rule.context);
119 }
else if (
rule.includeAttrib == XmlBool::True &&
rule.context.context) {
120 rule.context.context->referencedWithIncludeAttrib =
true;
125 auto *firstContext = &*definition.contexts.find(definition.firstContextName);
126 firstContext->isOnlyIncluded =
false;
127 definition.firstContext = firstContext;
130 resolveIncludeRules();
135 bool success = m_success;
137 const auto usedContexts = extractUsedContexts();
143 while (def.hasNext()) {
145 const auto &definition = def.value();
146 const auto &filename = definition.filename;
148 auto *maxDef = maxKateVersionDefinition(definition, maxVersionByDefinitions);
149 if (maxDef != &definition) {
150 qWarning() << definition.filename <<
"depends on a language" << maxDef->languageName <<
"in version" << maxDef->kateVersionStr
151 <<
". Please, increase kateversion.";
158 success = checkKeywordsList(definition, referencedKeywords) && success;
160 checkContexts(definition, referencedKeywords, usedAttributeNames, ignoredAttributeNames, usedContexts, unreachableIncludedRules) && success;
163 const auto invalidNames = usedAttributeNames - definition.itemDatas.styleNames;
164 for (
const auto &styleName : invalidNames) {
165 qWarning() << filename <<
"line" << styleName.line <<
"reference of non-existing itemData attributes:" << styleName.name;
170 const auto ignoredNames = ignoredAttributeNames - usedAttributeNames;
171 for (
const auto &styleName : ignoredNames) {
172 qWarning() << filename <<
"line" << styleName.line <<
"attribute" << styleName.name
173 <<
"is never used. All uses are with lookAhead=true or <IncludeRules/>";
178 auto unusedNames = definition.itemDatas.styleNames - usedAttributeNames;
179 unusedNames -= ignoredNames;
180 for (
const auto &styleName : std::as_const(unusedNames)) {
181 qWarning() << filename <<
"line" << styleName.line <<
"unused itemData:" << styleName.name;
187 while (unreachableIncludedRuleIt.hasNext()) {
188 unreachableIncludedRuleIt.next();
189 IncludedRuleUnreachableBy &unreachableRulesBy = unreachableIncludedRuleIt.value();
190 if (unreachableRulesBy.alwaysUnreachable) {
191 auto *
rule = unreachableIncludedRuleIt.key();
193 if (!
rule->parentContext->isOnlyIncluded) {
199 auto &unreachableBy = unreachableRulesBy.unreachableBy;
200 unreachableBy.
erase(std::remove_if(unreachableBy.begin(),
202 [&](
const RuleAndInclude &ruleAndInclude) {
203 if (rules.contains(ruleAndInclude.rule)) {
206 rules.
insert(ruleAndInclude.rule);
209 unreachableBy.end());
213 for (
auto &ruleAndInclude : std::as_const(unreachableBy)) {
214 message += QStringLiteral(
"line ");
216 message += QStringLiteral(
" [");
217 message += ruleAndInclude.rule->parentContext->name;
218 if (
rule->filename != ruleAndInclude.rule->filename) {
219 message += QStringLiteral(
" (");
220 message += ruleAndInclude.rule->filename;
223 if (ruleAndInclude.includeRules) {
224 message += QStringLiteral(
" via line ");
227 message += QStringLiteral(
"], ");
231 qWarning() <<
rule->filename <<
"line" <<
rule->line <<
"no IncludeRule can reach this rule, hidden by" <<
message;
253 Context *context =
nullptr;
266 if (attr.
name() != attrName) {
272 qWarning() << filename <<
"line" << xml.
lineNumber() << attrName <<
"attribute is empty";
281 bool extractXmlBool(XmlBool &xmlBool,
const QString &attrName)
283 if (attr.
name() != attrName) {
287 xmlBool = attr.
value().
isNull() ? XmlBool::Unspecified : attrToBool(attr.
value()) ? XmlBool::True : XmlBool::False;
294 bool extractPositive(
int &positive,
const QString &attrName)
296 if (attr.
name() != attrName) {
303 if (!ok || positive < 0) {
304 qWarning() << filename <<
"line" << xml.
lineNumber() << attrName <<
"should be a positive integer:" << attr.
value();
313 bool checkColor(
const QString &attrName)
315 if (attr.
name() != attrName) {
319 const auto value = attr.
value();
320 if (value.isEmpty() ) {
321 qWarning() << filename <<
"line" << xml.
lineNumber() << attrName <<
"should be a color:" << value;
332 if (attr.
name() != attrName) {
340 qWarning() << filename <<
"line" << xml.
lineNumber() << attrName <<
"must contain exactly one char:" << attr.
value();
348 bool checkIfExtracted(
bool isExtracted)
354 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"unknown attribute:" << attr.
name();
365 friend uint
qHash(
const Item &item, uint seed = 0)
367 return qHash(item.content, seed);
372 return item0.content == item1.content;
387 qWarning() << filename <<
"line" << line <<
"is empty:" << xml.
name();
391 if (xml.
name() == QStringLiteral(
"include")) {
392 includes.
insert({content, line});
393 }
else if (xml.
name() == QStringLiteral(
"item")) {
394 keywords.
append({content, line});
396 qWarning() << filename <<
"line" << line <<
"invalid element:" << xml.
name();
414 Parser parser{filename, xml, attr, success};
416 const bool isExtracted = parser.extractString(name, QStringLiteral(
"name"));
418 success = parser.checkIfExtracted(isExtracted);
450 bool isDotRegex =
false;
460 XmlBool firstNonSpace{};
463 XmlBool insensitive{};
472 XmlBool includeAttrib{};
494 Context
const *parentContext =
nullptr;
500 this->filename = filename;
504 static const auto pairs = {
505 Pair{QStringLiteral(
"AnyChar"), Type::AnyChar},
506 Pair{QStringLiteral(
"Detect2Chars"), Type::Detect2Chars},
507 Pair{QStringLiteral(
"DetectChar"), Type::DetectChar},
508 Pair{QStringLiteral(
"DetectIdentifier"), Type::DetectIdentifier},
509 Pair{QStringLiteral(
"DetectSpaces"), Type::DetectSpaces},
510 Pair{QStringLiteral(
"Float"), Type::Float},
511 Pair{QStringLiteral(
"HlCChar"), Type::HlCChar},
512 Pair{QStringLiteral(
"HlCHex"), Type::HlCHex},
513 Pair{QStringLiteral(
"HlCOct"), Type::HlCOct},
514 Pair{QStringLiteral(
"HlCStringChar"), Type::HlCStringChar},
515 Pair{QStringLiteral(
"IncludeRules"), Type::IncludeRules},
516 Pair{QStringLiteral(
"Int"), Type::Int},
517 Pair{QStringLiteral(
"LineContinue"), Type::LineContinue},
518 Pair{QStringLiteral(
"RangeDetect"), Type::RangeDetect},
519 Pair{QStringLiteral(
"RegExpr"), Type::RegExpr},
520 Pair{QStringLiteral(
"StringDetect"), Type::StringDetect},
521 Pair{QStringLiteral(
"WordDetect"), Type::WordDetect},
522 Pair{QStringLiteral(
"keyword"), Type::keyword},
525 for (
auto pair : pairs) {
526 if (xml.
name() == pair.first) {
528 bool success = parseAttributes(filename, xml);
529 success = checkMandoryAttributes(filename, xml) && success;
530 if (success && type == Type::RegExpr) {
532 static const QRegularExpression isDot(QStringLiteral(R
"(^\(?\.(?:[*+][*+?]?|[*+]|\{1\})?\$?$)"));
534 static const QRegularExpression removeParentheses(QStringLiteral(R
"(\((?:\?:)?|\))"));
540 static const QRegularExpression allSuffix(QStringLiteral(
"(?<!\\\\)[.][*][?+]?[$]?$"));
541 sanitizedString = string;
544 if (sanitizedString.
isEmpty() || sanitizedString == QStringLiteral(
"^")) {
545 sanitizedString = string;
552 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"unknown element:" << xml.
name();
562 Parser parser{filename, xml, attr, success};
565 const bool isExtracted
566 = parser.extractString(attribute, QStringLiteral(
"attribute"))
567 || parser.extractString(context.name, QStringLiteral(
"context"))
568 || parser.extractXmlBool(lookAhead, QStringLiteral(
"lookAhead"))
569 || parser.extractXmlBool(firstNonSpace, QStringLiteral(
"firstNonSpace"))
570 || parser.extractString(beginRegion, QStringLiteral(
"beginRegion"))
571 || parser.extractString(endRegion, QStringLiteral(
"endRegion"))
572 || parser.extractPositive(column, QStringLiteral(
"column"))
573 || ((
type == Type::RegExpr
574 ||
type == Type::StringDetect
575 ||
type == Type::WordDetect
576 ||
type == Type::keyword
577 ) && parser.extractXmlBool(insensitive, QStringLiteral(
"insensitive")))
578 || ((
type == Type::DetectChar
579 ||
type == Type::RegExpr
580 ||
type == Type::StringDetect
581 ||
type == Type::keyword
582 ) && parser.extractXmlBool(dynamic, QStringLiteral(
"dynamic")))
583 || ((
type == Type::RegExpr)
584 && parser.extractXmlBool(minimal, QStringLiteral(
"minimal")))
585 || ((
type == Type::DetectChar
586 ||
type == Type::Detect2Chars
587 ||
type == Type::LineContinue
588 ||
type == Type::RangeDetect
589 ) && parser.extractChar(char0, QStringLiteral(
"char")))
590 || ((
type == Type::Detect2Chars
591 ||
type == Type::RangeDetect
592 ) && parser.extractChar(char1, QStringLiteral(
"char1")))
593 || ((
type == Type::AnyChar
594 ||
type == Type::RegExpr
595 ||
type == Type::StringDetect
596 ||
type == Type::WordDetect
597 ||
type == Type::keyword
598 ) && parser.extractString(
string, QStringLiteral(
"String")))
599 || ((
type == Type::IncludeRules)
600 && parser.extractXmlBool(includeAttrib, QStringLiteral(
"includeAttrib")))
601 || ((
type == Type::Float
602 ||
type == Type::HlCHex
603 ||
type == Type::HlCOct
605 ||
type == Type::keyword
606 ||
type == Type::WordDetect
607 ) && (parser.extractString(additionalDeliminator, QStringLiteral(
"additionalDeliminator"))
608 || parser.extractString(weakDeliminator, QStringLiteral(
"weakDeliminator"))))
612 success = parser.checkIfExtracted(isExtracted);
614 if (type == Type::LineContinue && char0 ==
QLatin1Char(
'\0')) {
632 case Type::StringDetect:
633 case Type::WordDetect:
635 missingAttr =
string.
isEmpty() ? QStringLiteral(
"String") :
QString();
638 case Type::DetectChar:
639 missingAttr = !char0.
unicode() ? QStringLiteral(
"char") :
QString();
642 case Type::Detect2Chars:
643 case Type::RangeDetect:
644 missingAttr = !char0.
unicode() && !char1.
unicode() ? QStringLiteral(
"char and char1")
645 : !char0.unicode() ? QStringLiteral(
"char")
646 : !char1.unicode() ? QStringLiteral(
"char1")
650 case Type::IncludeRules:
651 missingAttr = context.name.isEmpty() ? QStringLiteral(
"context") :
QString();
654 case Type::DetectIdentifier:
655 case Type::DetectSpaces:
660 case Type::HlCStringChar:
662 case Type::LineContinue:
667 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"missing attribute:" << missingAttr;
677 bool isOnlyIncluded =
true;
679 bool referencedWithIncludeAttrib =
false;
680 bool hasDynamicRule =
false;
683 ContextName lineEndContext;
684 ContextName lineEmptyContext;
685 ContextName fallthroughContext;
688 XmlBool fallthrough{};
697 Parser parser{filename, xml, attr, success};
698 XmlBool noIndentationBasedFolding{};
700 const bool isExtracted = parser.extractString(name, QStringLiteral(
"name")) || parser.extractString(attribute, QStringLiteral(
"attribute"))
701 || parser.extractString(lineEndContext.name, QStringLiteral(
"lineEndContext"))
702 || parser.extractString(lineEmptyContext.name, QStringLiteral(
"lineEmptyContext"))
703 || parser.extractString(fallthroughContext.name, QStringLiteral(
"fallthroughContext"))
704 || parser.extractXmlBool(dynamic, QStringLiteral(
"dynamic")) || parser.extractXmlBool(fallthrough, QStringLiteral(
"fallthrough"))
705 || parser.extractXmlBool(noIndentationBasedFolding, QStringLiteral(
"noIndentationBasedFolding"));
707 success = parser.checkIfExtracted(isExtracted);
711 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"missing attribute: name";
716 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"missing attribute: attribute";
720 if (lineEndContext.name.isEmpty()) {
721 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"missing attribute: lineEndContext";
733 Version(
int majorRevision = 0,
int minorRevision = 0)
734 : majorRevision(majorRevision)
735 , minorRevision(minorRevision)
739 bool operator<(
const Version &version)
const
741 return majorRevision <
version.majorRevision || (majorRevision ==
version.majorRevision && minorRevision <
version.minorRevision);
750 friend uint
qHash(
const Style &style, uint seed = 0)
752 return qHash(style.name, seed);
755 friend bool operator==(
const Style &style0,
const Style &style1)
757 return style0.name == style1.name;
772 Parser parser{filename, xml, attr, success};
774 const bool isExtracted = parser.extractString(name, QStringLiteral(
"name")) || parser.extractString(defStyleNum, QStringLiteral(
"defStyleNum"))
775 || parser.extractXmlBool(
boolean, QStringLiteral(
"bold")) || parser.extractXmlBool(
boolean, QStringLiteral(
"italic"))
776 || parser.extractXmlBool(
boolean, QStringLiteral(
"underline")) || parser.extractXmlBool(
boolean, QStringLiteral(
"strikeOut"))
777 || parser.extractXmlBool(
boolean, QStringLiteral(
"spellChecking")) || parser.checkColor(QStringLiteral(
"color"))
778 || parser.checkColor(QStringLiteral(
"selColor")) || parser.checkColor(QStringLiteral(
"backgroundColor"))
779 || parser.checkColor(QStringLiteral(
"selBackgroundColor"));
781 success = parser.checkIfExtracted(isExtracted);
785 const auto len = styleNames.
size();
787 if (len == styleNames.
size()) {
788 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"itemData duplicate:" <<
name;
802 const Context *firstContext =
nullptr;
804 WordDelimiters wordDelimiters;
805 XmlBool casesensitive{};
814 wordDelimiters.append(xml.
attributes().
value(QStringLiteral(
"additionalDeliminator")));
824 m_success = context.parseElement(m_currentDefinition->filename, xml) && m_success;
825 if (m_currentDefinition->firstContextName.isEmpty()) {
826 m_currentDefinition->firstContextName = context.name;
828 if (m_currentDefinition->contexts.contains(context.name)) {
829 qWarning() << m_currentDefinition->filename <<
"line" << xml.
lineNumber() <<
"duplicate context:" << context.name;
832 m_currentContext = &*m_currentDefinition->contexts.insert(context.name, context);
839 m_success = keywords.parseElement(m_currentDefinition->filename, xml) && m_success;
840 if (m_currentDefinition->keywordsList.contains(keywords.name)) {
841 qWarning() << m_currentDefinition->filename <<
"line" << xml.
lineNumber() <<
"duplicate list:" << keywords.name;
844 m_currentKeywords = &*m_currentDefinition->keywordsList.insert(keywords.name, keywords);
849 auto it = maxVersionByDefinitions.
find(&definition);
850 if (it != maxVersionByDefinitions.
end()) {
853 auto it = maxVersionByDefinitions.
insert(&definition, &definition);
854 for (
const auto &referencedDef : definition.referencedDefinitions) {
855 auto *maxDef = maxKateVersionDefinition(*referencedDef, maxVersionByDefinitions);
856 if (it.value()->kateVersion < maxDef->kateVersion) {
865 void resolveIncludeRules()
871 while (def.hasNext()) {
873 auto &definition = def.value();
875 while (contextIt.hasNext()) {
877 auto ¤tContext = contextIt.value();
878 for (
auto &
rule : currentContext.rules) {
879 if (
rule.type != Context::Rule::Type::IncludeRules) {
883 if (
rule.context.stay) {
884 qWarning() << definition.filename <<
"line" <<
rule.line <<
"IncludeRules refers to himself";
889 if (
rule.context.popCount) {
890 qWarning() << definition.filename <<
"line" <<
rule.line <<
"IncludeRules with #pop prefix";
894 if (!
rule.context.context) {
901 usedContexts.
clear();
906 for (
int i = 0; i < contexts.
size(); ++i) {
907 currentContext.hasDynamicRule = contexts[i]->hasDynamicRule;
908 for (
const auto &includedRule : contexts[i]->rules) {
909 if (includedRule.type != Context::Rule::Type::IncludeRules) {
910 rule.includedRules.append(&includedRule);
911 }
else if (&
rule == &includedRule) {
912 qWarning() << definition.filename <<
"line" <<
rule.line <<
"IncludeRules refers to himself by recursivity";
915 rule.includedIncludeRules.insert(&includedRule);
917 if (includedRule.includedRules.isEmpty()) {
918 const auto *context = includedRule.context.context;
919 if (context && !usedContexts.
contains(context)) {
921 usedContexts.
insert(context);
924 rule.includedRules.append(includedRule.includedRules);
942 while (def.hasNext()) {
944 const auto &definition = def.value();
946 if (definition.firstContext) {
947 usedContexts.
insert(definition.firstContext);
949 contexts.
append(definition.firstContext);
951 for (
int i = 0; i < contexts.
size(); ++i) {
952 auto appendContext = [&](
const Context *context) {
953 if (context && !usedContexts.
contains(context)) {
955 usedContexts.
insert(context);
959 const auto *context = contexts[i];
960 appendContext(context->lineEndContext.context);
961 appendContext(context->lineEmptyContext.context);
962 appendContext(context->fallthroughContext.context);
964 for (
auto &
rule : context->rules) {
965 appendContext(
rule.context.context);
974 struct RuleAndInclude {
975 const Context::Rule *
rule;
976 const Context::Rule *includeRules;
978 explicit operator bool()
const
984 struct IncludedRuleUnreachableBy {
986 bool alwaysUnreachable =
true;
990 bool checkContexts(
const Definition &definition,
1000 while (contextIt.hasNext()) {
1003 const auto &context = contextIt.value();
1004 const auto &filename = definition.filename;
1006 if (!usedContexts.
contains(&context)) {
1007 qWarning() << filename <<
"line" << context.line <<
"unused context:" << context.name;
1012 if (context.name.startsWith(QStringLiteral(
"#pop"))) {
1013 qWarning() << filename <<
"line" << context.line <<
"the context name must not start with '#pop':" << context.name;
1017 if (!context.attribute.isEmpty() && (!context.isOnlyIncluded || context.referencedWithIncludeAttrib)) {
1018 usedAttributeNames.
insert({context.attribute, context.line});
1021 success = checkfallthrough(definition, context) && success;
1022 success = checkUreachableRules(definition.filename, context, unreachableIncludedRules) && success;
1023 success = suggestRuleMerger(definition.filename, context) && success;
1025 for (
const auto &
rule : context.rules) {
1026 if (!
rule.attribute.isEmpty()) {
1027 if (
rule.lookAhead != XmlBool::True) {
1033 success = checkLookAhead(
rule) && success;
1034 success = checkStringDetect(
rule) && success;
1035 success = checkKeyword(definition,
rule, referencedKeywords) && success;
1036 success = checkRegExpr(filename,
rule, context) && success;
1037 success = checkDelimiters(definition,
rule) && success;
1053 bool checkRegExpr(
const QString &filename,
const Context::Rule &
rule,
const Context &context)
const
1055 if (
rule.type == Context::Rule::Type::RegExpr) {
1057 if (!checkRegularExpression(
rule.filename, regexp,
rule.line)) {
1062 if (
rule.dynamic == XmlBool::True) {
1064 if (!
rule.string.contains(placeHolder)) {
1065 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"broken regex:" <<
rule.string <<
"problem: dynamic=true but no %\\d+ placeholder";
1070 auto reg = (
rule.lookAhead == XmlBool::True) ?
rule.sanitizedString :
rule.string;
1071 if (
rule.lookAhead == XmlBool::True) {
1073 R
"(((?<!\\)\\(?:[DSWdsw]|x[0-9a-fA-F]{2}|x\{[0-9a-fA-F]+\}|0\d\d|o\{[0-7]+\}|u[0-9a-fA-F]{4})|(?<!\\)[^])}\\]|(?=\\)\\\\)[*][?+]?$)"));
1074 reg.replace(removeAllSuffix, QString());
1077 reg.replace(QStringLiteral("{1}"),
QString());
1082 QStringLiteral(R
"(^\^?(?:\((?:\?:)?)?\^?(?:\\s|\[(?:\\s| (?:\t|\\t)|(?:\t|\\t) )\])\)?(?:[*+][*+?]?|[*+])?\)?\)?$)"));
1083 if (
rule.string.contains(isDetectSpaces)) {
1084 char const *extraMsg =
rule.string.contains(
QLatin1Char(
'^')) ?
"+ column=\"0\" or firstNonSpace=\"1\"" :
"";
1085 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"RegExpr should be replaced by DetectSpaces / DetectChar / AnyChar" << extraMsg <<
":"
1090 #define REG_ESCAPE_CHAR R"(\\(?:[^0BDPSWbdpswoux]|x[0-9a-fA-F]{2}|x\{[0-9a-fA-F]+\}|0\d\d|o\{[0-7]+\}|u[0-9a-fA-F]{4}))"
1091 #define REG_CHAR "(?:" REG_ESCAPE_CHAR "|\\[(?:" REG_ESCAPE_CHAR "|.)\\]|[^[.^])"
1095 "\\.\\*[?*]?" REG_CHAR
"|"
1096 "\\[\\^(" REG_ESCAPE_CHAR
"|.)\\]\\*[?*]?\\1"
1098 if ((
rule.lookAhead == XmlBool::True ||
rule.minimal == XmlBool::True ||
rule.string.contains(QStringLiteral(
".*?"))
1099 ||
rule.string.contains(QStringLiteral(
"[^")))
1100 && reg.contains(isRange)) {
1101 qWarning() << filename <<
"line" <<
rule.line <<
"RegExpr should be replaced by RangeDetect:" <<
rule.string;
1106 static const QRegularExpression isLineContinue(QStringLiteral(
"^\\^?" REG_CHAR
"\\$$"));
1107 if (reg.contains(isLineContinue)) {
1108 auto extra = (reg[0] ==
QLatin1Char(
'^')) ?
"with column=\"0\"" :
"";
1109 qWarning() << filename <<
"line" <<
rule.line <<
"RegExpr should be replaced by LineContinue:" <<
rule.string << extra;
1115 reg.replace(sanitize1, QStringLiteral(
"_"));
1118 #undef REG_ESCAPE_CHAR
1121 static const QRegularExpression isMinimal(QStringLiteral(
"(?![.][*+?][$]?[)]*$)[.][*+?][^?+]"));
1124 if (
rule.lookAhead == XmlBool::True &&
rule.minimal != XmlBool::True && reg.contains(isMinimal) && !reg.contains(hasNotGreedy)
1125 && (!
rule.context.context || !
rule.context.context->hasDynamicRule || regexp.captureCount() == 0)
1127 qWarning() << filename <<
"line" <<
rule.line
1128 <<
"RegExpr should be have minimal=\"1\" or use lazy operator (i.g, '.*' -> '.*?'):" <<
rule.string;
1134 reg.replace(sanitize2, QStringLiteral("___"));
1137 static const QRegularExpression sanitize3(QStringLiteral(R
"(\[(?:\^\]?[^]]*|\]?[^]\\]*?\\.[^]]*|\][^]]{2,}|[^]]{3,})\]|(\[\]?[^]]*\]))"));
1138 reg.replace(sanitize3, QStringLiteral("...\\1"));
1142 reg.replace(sanitize4, QStringLiteral("_"));
1144 const int len = reg.size();
1146 static const QRegularExpression toInsensitive(QStringLiteral(R
"(\[(?:([^]])\1)\])"));
1147 reg = reg.toUpper();
1148 reg.replace(toInsensitive, QString());
1152 static const QRegularExpression isStringDetect(QStringLiteral(R
"(^\^?(?:[^|\\?*+$^[{(.]|{(?!\d+,\d*}|,\d+})|\(\?:)+$)"));
1153 if (reg.contains(isStringDetect)) {
1154 char const *extraMsg =
rule.string.contains(
QLatin1Char(
'^')) ?
"+ column=\"0\" or firstNonSpace=\"1\"" :
"";
1155 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"RegExpr should be replaced by StringDetect / Detect2Chars / DetectChar" << extraMsg
1156 <<
":" <<
rule.string;
1157 if (len != reg.size()) {
1158 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"insensitive=\"1\" missing:" <<
rule.string;
1164 if (
rule.column == -1 &&
rule.firstNonSpace != XmlBool::True) {
1169 auto first = std::as_const(reg).begin();
1170 auto last = std::as_const(reg).end();
1182 const int bolDepth = depth;
1185 while (++first != last) {
1190 if (depth < bolDepth) {
1192 if (first + 1 != last && QStringLiteral(
"*?").contains(first[1])) {
1199 if (depth <= bolDepth) {
1207 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"column=\"0\" or firstNonSpace=\"1\" missing with RegExpr:" <<
rule.string;
1214 if (
rule.column == 0 && !
rule.isDotRegex) {
1215 bool hasStartOfLine =
false;
1216 auto first = std::as_const(reg).begin();
1217 auto last = std::as_const(reg).end();
1218 for (; first != last; ++first) {
1220 hasStartOfLine =
true;
1231 if (!hasStartOfLine) {
1232 qWarning() <<
rule.filename <<
"line" <<
rule.line
1233 <<
"start of line missing in the pattern with column=\"0\" (i.e. abc -> ^abc):" <<
rule.string;
1238 bool useCapture =
false;
1241 if (regexp.captureCount()) {
1242 auto maximalCapture = [](
const QString(&referenceNames)[9],
const QString &s) {
1244 while (maxCapture && !s.contains(referenceNames[maxCapture - 1])) {
1250 int maxCaptureUsed = 0;
1252 if (
rule.context.context && !
rule.context.stay) {
1253 for (
const auto &nextRule :
rule.context.context->rules) {
1254 if (nextRule.dynamic == XmlBool::True) {
1256 QStringLiteral(
"%1"),
1257 QStringLiteral(
"%2"),
1258 QStringLiteral(
"%3"),
1259 QStringLiteral(
"%4"),
1260 QStringLiteral(
"%5"),
1261 QStringLiteral(
"%6"),
1262 QStringLiteral(
"%7"),
1263 QStringLiteral(
"%8"),
1264 QStringLiteral(
"%9"),
1266 int maxDynamicCapture = maximalCapture(cap, nextRule.string);
1267 maxCaptureUsed = std::max(maxCaptureUsed, maxDynamicCapture);
1273 QStringLiteral(
"\\1"),
1274 QStringLiteral(
"\\2"),
1275 QStringLiteral(
"\\3"),
1276 QStringLiteral(
"\\4"),
1277 QStringLiteral(
"\\5"),
1278 QStringLiteral(
"\\6"),
1279 QStringLiteral(
"\\7"),
1280 QStringLiteral(
"\\8"),
1281 QStringLiteral(
"\\9"),
1284 QStringLiteral(
"\\g1"),
1285 QStringLiteral(
"\\g2"),
1286 QStringLiteral(
"\\g3"),
1287 QStringLiteral(
"\\g4"),
1288 QStringLiteral(
"\\g5"),
1289 QStringLiteral(
"\\g6"),
1290 QStringLiteral(
"\\g7"),
1291 QStringLiteral(
"\\g8"),
1292 QStringLiteral(
"\\g9"),
1294 const int maxBackReference = std::max(maximalCapture(num1,
rule.string), maximalCapture(num1,
rule.string));
1296 const int maxCapture = std::max(maxCaptureUsed, maxBackReference);
1298 if (maxCapture && regexp.captureCount() > maxCapture) {
1299 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"RegExpr with" << regexp.captureCount() <<
"captures but only" << maxCapture
1300 <<
"are used. Please, replace '(...)' with '(?:...)':" <<
rule.string;
1304 useCapture = maxCapture;
1310 QStringLiteral(R
"(^(\((\?:)?)?\[((a-z|_){2}|(A-Z|_){2})\]([+][*?]?)?\[((0-9|a-z|_){3}|(0-9|A-Z|_){3})\][*][*?]?(\))?$)"));
1312 QStringLiteral(R
"(^(\((\?:)?)?\[(a-z|A-Z|_){3}\]([+][*?]?)?\[(0-9|a-z|A-Z|_){4}\][*][*?]?(\))?$)"));
1313 auto &isDetectIdentifier = (
rule.insensitive == XmlBool::True) ? isInsensitiveDetectIdentifier : isSensitiveDetectIdentifier;
1314 if (
rule.string.contains(isDetectIdentifier)) {
1315 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"RegExpr should be replaced by DetectIdentifier:" <<
rule.string;
1320 if (
rule.isDotRegex) {
1322 int i = &
rule - context.rules.data() + 1;
1323 const bool hasColumn = (
rule.column != -1);
1324 const bool hasFirstNonSpace = (
rule.firstNonSpace == XmlBool::True);
1325 const bool isSpecial = (hasColumn || hasFirstNonSpace);
1326 for (; i < context.rules.size(); ++i) {
1327 auto &rule2 = context.rules[i];
1328 if (rule2.type == Context::Rule::Type::IncludeRules && isSpecial) {
1329 i = context.rules.size();
1333 const bool hasColumn2 = (rule2.column != -1);
1334 const bool hasFirstNonSpace2 = (rule2.firstNonSpace == XmlBool::True);
1335 if ((!isSpecial && !hasColumn2 && !hasFirstNonSpace2) || (hasColumn &&
rule.column == rule2.column)
1336 || (hasFirstNonSpace && hasFirstNonSpace2)) {
1341 auto ruleFilename = (filename ==
rule.filename) ?
QString() : QStringLiteral(
"in ") +
rule.filename;
1342 if (i == context.rules.size()) {
1343 if (
rule.lookAhead == XmlBool::True &&
rule.firstNonSpace != XmlBool::True &&
rule.column == -1 &&
rule.beginRegion.isEmpty()
1344 &&
rule.endRegion.isEmpty() && !useCapture) {
1345 qWarning() << filename <<
"context line" << context.line <<
": RegExpr line" <<
rule.line << ruleFilename
1346 <<
"should be replaced by fallthroughContext:" <<
rule.string;
1349 auto &nextRule = context.rules[i];
1350 auto nextRuleFilename = (filename == nextRule.filename) ?
QString() : QStringLiteral(
"in ") + nextRule.filename;
1351 qWarning() << filename <<
"context line" << context.line <<
"contains unreachable element line" << nextRule.line << nextRuleFilename
1352 <<
"because a dot RegExpr is used line" <<
rule.line << ruleFilename;
1356 static const QRegularExpression unnecessaryQuantifier1(QStringLiteral(R
"([*+?]([.][*+?]{0,2})?$)"));
1357 static const QRegularExpression unnecessaryQuantifier2(QStringLiteral(R
"([*+?]([.][*+?]{0,2})?[)]*$)"));
1358 auto &unnecessaryQuantifier = useCapture ? unnecessaryQuantifier1 : unnecessaryQuantifier2;
1359 if (
rule.lookAhead == XmlBool::True &&
rule.minimal != XmlBool::True && reg.contains(unnecessaryQuantifier)) {
1360 qWarning() << filename <<
"line" <<
rule.line
1361 <<
"Last quantifier is not necessary (i.g., 'xyz*' -> 'xy', 'xyz+.' -> 'xyz.'):" <<
rule.string;
1373 bool success =
true;
1376 XmlBool casesensitive{};
1379 Parser parser{filename, xml, attr, success};
1381 const bool isExtracted =
1382 parser.extractString(pattern, QStringLiteral(
"regexpr")) || parser.extractXmlBool(casesensitive, QStringLiteral(
"casesensitive"));
1384 success = parser.checkIfExtracted(isExtracted);
1388 qWarning() << filename <<
"line" << xml.
lineNumber() <<
"missing attribute: regexpr";
1406 qWarning() << filename <<
"line" << line <<
"broken regex:" <<
pattern <<
"problem:" << regexp.
errorString() <<
"at offset"
1413 if (azOffset >= 0) {
1414 qWarning() << filename <<
"line" << line <<
"broken regex:" <<
pattern <<
"problem: [a-Z] or [A-z] at offset" << azOffset;
1423 bool checkfallthrough(
const Definition &definition,
const Context &context)
const
1425 bool success =
true;
1427 if (!context.fallthroughContext.name.isEmpty()) {
1428 if (context.fallthroughContext.stay) {
1429 qWarning() << definition.filename <<
"line" << context.line <<
"possible infinite loop due to fallthroughContext=\"#stay\" in context "
1434 const bool mandatoryFallthroughAttribute = definition.kateVersion <
Version{5, 62};
1435 if (context.fallthrough == XmlBool::True && !mandatoryFallthroughAttribute) {
1436 qWarning() << definition.filename <<
"line" << context.line <<
"fallthrough attribute is unnecessary with kateversion >= 5.62 in context"
1439 }
else if (context.fallthrough != XmlBool::True && mandatoryFallthroughAttribute) {
1440 qWarning() << definition.filename <<
"line" << context.line
1441 <<
"fallthroughContext attribute without fallthrough=\"1\" attribute is only valid with kateversion >= 5.62 in context"
1451 bool checkDelimiters(
const Definition &definition,
const Context::Rule &
rule)
const
1453 if (
rule.additionalDeliminator.isEmpty() &&
rule.weakDeliminator.isEmpty()) {
1457 bool success =
true;
1459 if (definition.kateVersion < Version{5, 79}) {
1460 qWarning() << definition.filename <<
"line" <<
rule.line
1461 <<
"additionalDeliminator and weakDeliminator are only available since version \"5.79\". Please, increase kateversion.";
1465 for (
QChar c :
rule.additionalDeliminator) {
1466 if (!definition.wordDelimiters.contains(c)) {
1472 if (definition.wordDelimiters.contains(c)) {
1477 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"unnecessary use of additionalDeliminator and/or weakDeliminator" <<
rule.string;
1485 if (
rule.type == Context::Rule::Type::keyword) {
1486 auto it = definition.keywordsList.find(
rule.string);
1487 if (it != definition.keywordsList.end()) {
1488 referencedKeywords.
insert(&*it);
1490 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"reference of non-existing keyword list:" <<
rule.string;
1499 bool checkLookAhead(
const Context::Rule &
rule)
const
1501 if (
rule.lookAhead == XmlBool::True &&
rule.context.stay) {
1502 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"infinite loop: lookAhead with context #stay";
1514 bool checkStringDetect(
const Context::Rule &
rule)
const
1516 if (
rule.type == Context::Rule::Type::StringDetect) {
1518 if (
rule.dynamic == XmlBool::True) {
1520 if (!
rule.string.contains(placeHolder)) {
1521 qWarning() <<
rule.filename <<
"line" <<
rule.line <<
"broken regex:" <<
rule.string <<
"problem: dynamic=true but no %\\d+ placeholder";
1532 bool success =
true;
1534 bool includeNotSupport = (definition.kateVersion <
Version{5, 53});
1536 while (keywordsIt.hasNext()) {
1539 for (
const auto &include : keywordsIt.value().items.includes) {
1540 if (includeNotSupport) {
1541 qWarning() << definition.filename <<
"line" << include.line
1542 <<
"<include> is only available since version \"5.53\". Please, increase kateversion.";
1545 success = checkKeywordInclude(definition, include, referencedKeywords) && success;
1550 for (
const auto& keyword : keywordsIt.value().items.keywords) {
1551 for (
QChar c : keyword.content) {
1552 if (definition.wordDelimiters.contains(c)) {
1553 qWarning() << definition.filename <<
"line" << keyword.line <<
"keyword with delimiter:" << c <<
"in" << keyword.content;
1565 bool checkKeywordInclude(
const Definition &definition,
const Keywords::Items::Item &include,
QSet<const Keywords *> &referencedKeywords)
const
1567 bool containsKeywordName =
true;
1568 int const idx = include.content.indexOf(QStringLiteral(
"##"));
1570 auto it = definition.keywordsList.find(include.content);
1571 containsKeywordName = (it != definition.keywordsList.end());
1572 if (containsKeywordName) {
1573 referencedKeywords.
insert(&*it);
1576 auto defName = include.content.mid(idx + 2);
1577 auto listName = include.content.left(idx);
1578 auto it = m_definitions.find(defName);
1579 if (it == m_definitions.end()) {
1580 qWarning() << definition.filename <<
"line" << include.line <<
"unknown definition in" << include.content;
1583 containsKeywordName = it->keywordsList.contains(listName);
1586 if (!containsKeywordName) {
1587 qWarning() << definition.filename <<
"line" << include.line <<
"unknown keyword name in" << include.content;
1590 return containsKeywordName;
1599 bool checkUreachableRules(
const QString &filename,
1600 const Context &context,
1603 if (context.isOnlyIncluded) {
1608 RuleAndInclude setRule(
const Context::Rule &
rule,
const Context::Rule *includeRules =
nullptr)
1610 auto set = [&](RuleAndInclude &ruleAndInclude) {
1611 auto old = ruleAndInclude;
1612 ruleAndInclude = {&
rule, includeRules};
1616 if (
rule.firstNonSpace == XmlBool::True) {
1617 return set(firstNonSpace);
1618 }
else if (
rule.column == 0) {
1619 return set(column0);
1620 }
else if (
rule.column > 0) {
1621 return set(columnGreaterThan0[
rule.column]);
1628 RuleAndInclude normal;
1629 RuleAndInclude column0;
1631 RuleAndInclude firstNonSpace;
1640 return m_asciiMap[c.
unicode()];
1642 auto it = m_utf8Map.find(c);
1643 return it == m_utf8Map.end() ? RuleAndInclude{
nullptr,
nullptr} : it.value();
1666 void append(
QChar c,
const Context::Rule &
rule,
const Context::Rule *includeRule =
nullptr)
1671 m_utf8Map[c] = {&
rule, includeRule};
1676 void append(
QStringView s,
const Context::Rule &
rule,
const Context::Rule *includeRule =
nullptr)
1679 append(c,
rule, includeRule);
1684 RuleAndInclude m_asciiMap[127]{};
1688 struct Char4Tables {
1690 CharTable charsColumn0;
1692 CharTable charsFirstNonSpace;
1696 struct CharTableArray {
1699 CharTableArray(Char4Tables &tables,
const Context::Rule &
rule)
1701 if (
rule.firstNonSpace == XmlBool::True) {
1702 appendTable(tables.charsFirstNonSpace);
1705 if (
rule.column == 0) {
1706 appendTable(tables.charsColumn0);
1707 }
else if (
rule.column > 0) {
1708 appendTable(tables.charsColumnGreaterThan0[
rule.column]);
1711 appendTable(tables.chars);
1715 void removeNonSpecialWhenSpecial()
1725 for (
int i = 0; i < m_size; ++i) {
1726 if (
auto ruleAndInclude = m_charTables[i]->
find(c)) {
1727 return ruleAndInclude;
1730 return RuleAndInclude{
nullptr,
nullptr};
1737 for (
int i = 0; i < m_size; ++i) {
1738 auto result = m_charTables[i]->find(s);
1739 if (result.
size()) {
1740 while (++i < m_size) {
1750 void append(
QChar c,
const Context::Rule &
rule,
const Context::Rule *includeRule =
nullptr)
1752 for (
int i = 0; i < m_size; ++i) {
1753 m_charTables[i]->append(c,
rule, includeRule);
1758 void append(
QStringView s,
const Context::Rule &
rule,
const Context::Rule *includeRule =
nullptr)
1760 for (
int i = 0; i < m_size; ++i) {
1761 m_charTables[i]->append(s,
rule, includeRule);
1766 void appendTable(CharTable &t)
1768 m_charTables[m_size] = &t;
1772 CharTable *m_charTables[3];
1776 struct ObservableRule {
1777 const Context::Rule *
rule;
1778 const Context::Rule *includeRules;
1780 bool hasResolvedIncludeRules()
const
1782 return rule == includeRules;
1787 struct RuleIterator {
1789 : m_end(&endRule - rules.data())
1795 const Context::Rule *
next()
1798 if (m_includedRules) {
1800 if (m_i2 != m_includedRules->size()) {
1801 return (*m_includedRules)[m_i2];
1804 m_includedRules =
nullptr;
1808 while (m_i < m_end && m_rules[m_i].
rule->type == Context::Rule::Type::IncludeRules) {
1809 if (!m_rules[m_i].includeRules && m_rules[m_i].
rule->includedRules.size()) {
1811 m_includedRules = &m_rules[m_i].rule->includedRules;
1812 return (*m_includedRules)[m_i2];
1819 return m_rules[m_i - 1].rule;
1826 const Context::Rule *currentIncludeRules()
const
1828 return m_includedRules ? m_rules[m_i].rule : m_rules[m_i].includeRules;
1842 void append(
const Context::Rule &
rule,
const Context::Rule *includedRule)
1844 auto array = extractDotRegexes(
rule);
1846 *array[0] = {&
rule, includedRule};
1849 *array[1] = {&
rule, includedRule};
1854 RuleAndInclude
find(
const Context::Rule &
rule)
1856 auto array = extractDotRegexes(
rule);
1863 return RuleAndInclude{};
1867 using Array = std::array<RuleAndInclude *, 2>;
1869 Array extractDotRegexes(
const Context::Rule &
rule)
1873 if (
rule.firstNonSpace != XmlBool::True &&
rule.column == -1) {
1876 if (
rule.firstNonSpace == XmlBool::True) {
1877 ret[0] = &dotRegexFirstNonSpace;
1880 if (
rule.column == 0) {
1881 ret[1] = &dotRegexColumn0;
1882 }
else if (
rule.column > 0) {
1883 ret[1] = &dotRegexColumnGreaterThan0[
rule.column];
1890 RuleAndInclude dotRegex{};
1891 RuleAndInclude dotRegexColumn0{};
1893 RuleAndInclude dotRegexFirstNonSpace{};
1896 bool success =
true;
1899 Char4Tables detectChars;
1901 Char4Tables dynamicDetectChars;
1903 Char4Tables lineContinueChars;
1907 Rule4 hlCCharRule{};
1910 Rule4 hlCStringCharRule{};
1911 Rule4 detectIdentifierRule{};
1919 observedRules.
reserve(context.rules.size());
1920 for (
const Context::Rule &
rule : context.rules) {
1921 const Context::Rule *includeRule =
nullptr;
1922 if (
rule.type == Context::Rule::Type::IncludeRules) {
1923 auto *context =
rule.context.context;
1924 if (context && context->isOnlyIncluded) {
1925 includeRule = &
rule;
1931 for (
const Context::Rule *rule2 :
rule.includedRules) {
1932 observedRules.
push_back({rule2, includeRule});
1937 for (
auto &observedRule : observedRules) {
1938 const Context::Rule &
rule = *observedRule.rule;
1939 bool isUnreachable =
false;
1943 auto updateUnreachable1 = [&](RuleAndInclude ruleAndInclude) {
1944 if (ruleAndInclude) {
1945 isUnreachable =
true;
1946 unreachableBy.
append(ruleAndInclude);
1952 if (!ruleAndIncludes.isEmpty()) {
1953 isUnreachable =
true;
1954 unreachableBy.
append(ruleAndIncludes);
1959 auto isCompatible = [&
rule](Context::Rule
const &rule2) {
1960 return (rule2.firstNonSpace != XmlBool::True && rule2.column == -1) || (
rule.column == rule2.column &&
rule.column != -1)
1961 || (
rule.firstNonSpace == rule2.firstNonSpace &&
rule.firstNonSpace == XmlBool::True);
1964 updateUnreachable1(dotRegex.find(
rule));
1966 switch (
rule.type) {
1969 case Context::Rule::Type::AnyChar: {
1970 auto tables = CharTableArray(detectChars,
rule);
1971 updateUnreachable2(tables.find(
rule.string));
1972 tables.removeNonSpecialWhenSpecial();
1979 case Context::Rule::Type::DetectChar: {
1980 auto &chars4 = (
rule.dynamic != XmlBool::True) ? detectChars : dynamicDetectChars;
1981 auto tables = CharTableArray(chars4,
rule);
1982 updateUnreachable1(tables.find(
rule.char0));
1983 tables.removeNonSpecialWhenSpecial();
1990 case Context::Rule::Type::DetectSpaces: {
1991 auto tables = CharTableArray(detectChars,
rule);
1992 updateUnreachable2(tables.find(QStringLiteral(
" \t")));
1993 tables.removeNonSpecialWhenSpecial();
2000 case Context::Rule::Type::HlCChar:
2002 updateUnreachable1(hlCCharRule.setRule(
rule));
2006 case Context::Rule::Type::HlCHex:
2008 updateUnreachable1(hlCHexRule.setRule(
rule));
2012 case Context::Rule::Type::HlCOct:
2014 updateUnreachable1(hlCOctRule.setRule(
rule));
2018 case Context::Rule::Type::HlCStringChar:
2020 updateUnreachable1(hlCStringCharRule.setRule(
rule));
2024 case Context::Rule::Type::Int:
2025 updateUnreachable2(CharTableArray(detectChars,
rule).
find(QStringLiteral(
"0123456789")));
2026 updateUnreachable1(intRule.setRule(
rule));
2030 case Context::Rule::Type::Float:
2031 updateUnreachable2(CharTableArray(detectChars,
rule).
find(QStringLiteral(
"0123456789.")));
2032 updateUnreachable1(floatRule.setRule(
rule));
2036 case Context::Rule::Type::DetectIdentifier:
2037 updateUnreachable1(detectIdentifierRule.setRule(
rule));
2041 case Context::Rule::Type::LineContinue: {
2042 updateUnreachable1(CharTableArray(detectChars,
rule).
find(
rule.char0));
2044 auto tables = CharTableArray(lineContinueChars,
rule);
2045 updateUnreachable1(tables.find(
rule.char0));
2046 tables.removeNonSpecialWhenSpecial();
2052 case Context::Rule::Type::Detect2Chars:
2053 case Context::Rule::Type::RangeDetect:
2054 updateUnreachable1(CharTableArray(detectChars,
rule).
find(
rule.char0));
2055 if (!isUnreachable) {
2056 RuleIterator ruleIterator(observedRules, observedRule);
2057 while (
const auto *rulePtr = ruleIterator.next()) {
2058 if (isUnreachable) {
2061 const auto &rule2 = *rulePtr;
2062 if (rule2.type ==
rule.type && isCompatible(rule2) &&
rule.char0 == rule2.char0 &&
rule.char1 == rule2.char1) {
2063 updateUnreachable1({&rule2, ruleIterator.currentIncludeRules()});
2069 case Context::Rule::Type::RegExpr: {
2070 if (
rule.isDotRegex) {
2071 dotRegex.append(
rule,
nullptr);
2076 RuleIterator ruleIterator(observedRules, observedRule);
2077 while (
const auto *rulePtr = ruleIterator.next()) {
2078 if (isUnreachable) {
2081 const auto &rule2 = *rulePtr;
2082 if (rule2.type == Context::Rule::Type::RegExpr && isCompatible(rule2) &&
rule.insensitive == rule2.insensitive
2083 &&
rule.dynamic == rule2.dynamic &&
rule.sanitizedString.startsWith(rule2.sanitizedString)) {
2084 bool add = (
rule.sanitizedString.startsWith(rule2.string) ||
rule.sanitizedString.size() < rule2.sanitizedString.size() + 2);
2088 auto c1 =
rule.sanitizedString[rule2.sanitizedString.size()].unicode();
2089 auto c2 =
rule.sanitizedString[rule2.sanitizedString.size() + 1].unicode();
2090 auto c3 = rule2.sanitizedString.back().unicode();
2091 if (c3 ==
'*' || c3 ==
'?' || c3 ==
'+') {
2093 }
else if (c1 ==
'*' || c1 ==
'?') {
2094 add = !((c2 ==
'?' || c2 ==
'+') || (
rule.sanitizedString.size() >= rule2.sanitizedString.size() + 3));
2100 updateUnreachable1({&rule2, ruleIterator.currentIncludeRules()});
2108 case Context::Rule::Type::WordDetect:
2109 case Context::Rule::Type::StringDetect: {
2111 if (
rule.type == Context::Rule::Type::StringDetect &&
rule.dynamic == XmlBool::True) {
2112 RuleIterator ruleIterator(observedRules, observedRule);
2113 while (
const auto *rulePtr = ruleIterator.next()) {
2114 if (isUnreachable) {
2118 const auto &rule2 = *rulePtr;
2119 if (rule2.type != Context::Rule::Type::StringDetect || rule2.dynamic != XmlBool::True || !isCompatible(rule2)) {
2123 const bool isSensitive = (rule2.insensitive == XmlBool::True);
2125 if ((isSensitive ||
rule.insensitive != XmlBool::True) &&
rule.string.startsWith(rule2.string, caseSensitivity)) {
2126 updateUnreachable1({&rule2, ruleIterator.currentIncludeRules()});
2135 if (
rule.dynamic == XmlBool::True) {
2136 static const QRegularExpression dynamicPosition(QStringLiteral(R
"(^(?:[^%]*|%(?![1-9]))*)"));
2137 auto result = dynamicPosition.match(
rule.string);
2138 s = s.
left(result.capturedLength());
2144 if (
rule.type == Context::Rule::Type::RegExpr) {
2145 static const QRegularExpression regularChars(QStringLiteral(R
"(^(?:[^.?*+^$[{(\\|]+|\\[-.?*+^$[\]{}()\\|]+|\[[^^\\]\])+)"));
2146 static const QRegularExpression sanitizeChars(QStringLiteral(R
"(\\([-.?*+^$[\]{}()\\|])|\[([^^\\])\])"));
2147 const qsizetype result = regularChars.match(
rule.string).capturedLength();
2148 const qsizetype pos = qMin(result, s.
size());
2150 sanitizedRegex =
rule.string.left(qMin(result, s.
size()));
2151 sanitizedRegex.
replace(sanitizeChars, QStringLiteral(
"\\1"));
2160 auto t = CharTableArray(detectChars,
rule);
2161 if (
rule.insensitive != XmlBool::True) {
2162 updateUnreachable1(t.find(s[0]));
2164 QChar c2[]{s[0].toLower(), s[0].toUpper()};
2170 if (s.
size() > 0 && !isUnreachable) {
2172 RuleAndInclude detect2CharsInsensitives[]{{}, {}, {}, {}};
2174 RuleIterator ruleIterator(observedRules, observedRule);
2175 while (
const auto *rulePtr = ruleIterator.next()) {
2176 if (isUnreachable) {
2179 const auto &rule2 = *rulePtr;
2180 const bool isSensitive = (rule2.insensitive == XmlBool::True);
2183 switch (rule2.type) {
2185 case Context::Rule::Type::Detect2Chars:
2186 if (isCompatible(rule2) && s.
size() >= 2) {
2187 if (
rule.insensitive != XmlBool::True) {
2188 if (rule2.char0 == s[0] && rule2.char1 == s[1]) {
2189 updateUnreachable1({&rule2, ruleIterator.currentIncludeRules()});
2194 auto set = [&](RuleAndInclude &x,
QChar c1,
QChar c2) {
2195 if (!x && rule2.char0 == c1 && rule2.char0 == c2) {
2196 x = {&rule2, ruleIterator.currentIncludeRules()};
2199 set(detect2CharsInsensitives[0], s[0].toLower(), s[1].toLower());
2200 set(detect2CharsInsensitives[1], s[0].toLower(), s[1].toUpper());
2201 set(detect2CharsInsensitives[2], s[0].toUpper(), s[1].toUpper());
2202 set(detect2CharsInsensitives[3], s[0].toUpper(), s[1].toLower());
2204 if (detect2CharsInsensitives[0] && detect2CharsInsensitives[1] && detect2CharsInsensitives[2]
2205 && detect2CharsInsensitives[3]) {
2206 isUnreachable =
true;
2207 unreachableBy.
append(detect2CharsInsensitives[0]);
2208 unreachableBy.
append(detect2CharsInsensitives[1]);
2209 unreachableBy.
append(detect2CharsInsensitives[2]);
2210 unreachableBy.
append(detect2CharsInsensitives[3]);
2217 case Context::Rule::Type::StringDetect:
2218 if (isCompatible(rule2) && rule2.dynamic != XmlBool::True && (isSensitive ||
rule.insensitive != XmlBool::True)
2219 && s.
startsWith(rule2.string, caseSensitivity)) {
2220 updateUnreachable1({&rule2, ruleIterator.currentIncludeRules()});
2225 case Context::Rule::Type::WordDetect:
2226 if (
rule.type == Context::Rule::Type::WordDetect && isCompatible(rule2) && (isSensitive ||
rule.insensitive != XmlBool::True)
2227 && 0 ==
rule.string.compare(rule2.string, caseSensitivity)) {
2228 updateUnreachable1({&rule2, ruleIterator.currentIncludeRules()});
2241 case Context::Rule::Type::keyword: {
2242 RuleIterator ruleIterator(observedRules, observedRule);
2243 while (
const auto *rulePtr = ruleIterator.next()) {
2244 if (isUnreachable) {
2247 const auto &rule2 = *rulePtr;
2248 if (rule2.type == Context::Rule::Type::keyword && isCompatible(rule2) &&
rule.string == rule2.string) {
2249 updateUnreachable1({&rule2, ruleIterator.currentIncludeRules()});
2261 case Context::Rule::Type::IncludeRules:
2262 if (observedRule.includeRules && !observedRule.hasResolvedIncludeRules()) {
2266 if (
auto &ruleAndInclude = includeContexts[
rule.context.context]) {
2267 updateUnreachable1(ruleAndInclude);
2269 ruleAndInclude.rule = &
rule;
2272 for (
const auto *rulePtr :
rule.includedIncludeRules) {
2273 includeContexts.
insert(rulePtr->context.context, RuleAndInclude{rulePtr, &rule});
2276 if (observedRule.includeRules) {
2280 for (
const auto *rulePtr :
rule.includedRules) {
2281 const auto &rule2 = *rulePtr;
2282 switch (rule2.type) {
2283 case Context::Rule::Type::AnyChar: {
2284 auto tables = CharTableArray(detectChars, rule2);
2285 tables.removeNonSpecialWhenSpecial();
2286 tables.append(rule2.string, rule2, &
rule);
2290 case Context::Rule::Type::DetectChar: {
2291 auto &chars4 = (
rule.dynamic != XmlBool::True) ? detectChars : dynamicDetectChars;
2292 auto tables = CharTableArray(chars4, rule2);
2293 tables.removeNonSpecialWhenSpecial();
2294 tables.append(rule2.char0, rule2, &
rule);
2298 case Context::Rule::Type::DetectSpaces: {
2299 auto tables = CharTableArray(detectChars, rule2);
2300 tables.removeNonSpecialWhenSpecial();
2306 case Context::Rule::Type::HlCChar:
2307 hlCCharRule.setRule(rule2, &
rule);
2310 case Context::Rule::Type::HlCHex:
2311 hlCHexRule.setRule(rule2, &
rule);
2314 case Context::Rule::Type::HlCOct:
2315 hlCOctRule.setRule(rule2, &
rule);
2318 case Context::Rule::Type::HlCStringChar:
2319 hlCStringCharRule.setRule(rule2, &
rule);
2322 case Context::Rule::Type::Int:
2323 intRule.setRule(rule2, &
rule);
2326 case Context::Rule::Type::Float:
2327 floatRule.setRule(rule2, &
rule);
2330 case Context::Rule::Type::LineContinue: {
2331 auto tables = CharTableArray(lineContinueChars, rule2);
2332 tables.removeNonSpecialWhenSpecial();
2333 tables.append(rule2.char0, rule2, &
rule);
2337 case Context::Rule::Type::RegExpr:
2338 if (rule2.isDotRegex) {
2339 dotRegex.append(rule2, &
rule);
2343 case Context::Rule::Type::WordDetect:
2344 case Context::Rule::Type::StringDetect:
2345 case Context::Rule::Type::Detect2Chars:
2346 case Context::Rule::Type::IncludeRules:
2347 case Context::Rule::Type::DetectIdentifier:
2348 case Context::Rule::Type::keyword:
2349 case Context::Rule::Type::Unknown:
2350 case Context::Rule::Type::RangeDetect:
2356 case Context::Rule::Type::Unknown:
2360 if (observedRule.includeRules && !observedRule.hasResolvedIncludeRules()) {
2361 auto &unreachableIncludedRule = unreachableIncludedRules[&
rule];
2362 if (isUnreachable && unreachableIncludedRule.alwaysUnreachable) {
2363 unreachableIncludedRule.unreachableBy.append(unreachableBy);
2365 unreachableIncludedRule.alwaysUnreachable =
false;
2367 }
else if (isUnreachable) {
2371 for (
auto &ruleAndInclude : unreachableBy) {
2372 message += QStringLiteral(
"line ");
2373 if (ruleAndInclude.includeRules) {
2375 message += QStringLiteral(
" [by '");
2376 message += ruleAndInclude.includeRules->context.name;
2377 message += QStringLiteral(
"' line ");
2379 if (ruleAndInclude.includeRules->filename != ruleAndInclude.rule->filename) {
2380 message += QStringLiteral(
" (");
2381 message += ruleAndInclude.rule->filename;
2388 message += QStringLiteral(
", ");
2391 qWarning() << filename <<
"line" <<
rule.line <<
"unreachable rule by" <<
message;
2401 bool suggestRuleMerger(
const QString &filename,
const Context &context)
const
2403 bool success =
true;
2405 if (context.rules.isEmpty()) {
2409 auto it = context.rules.begin();
2410 const auto end = context.rules.end() - 1;
2412 for (; it <
end; ++it) {
2414 auto &rule2 = it[1];
2416 auto isCommonCompatible = [&] {
2417 if (rule1.lookAhead != rule2.lookAhead) {
2421 if (rule1.lookAhead != XmlBool::True && rule1.attribute != rule2.attribute) {
2425 return rule1.beginRegion == rule2.beginRegion
2426 && rule1.endRegion == rule2.endRegion
2427 && rule1.firstNonSpace == rule2.firstNonSpace
2428 && rule1.context.context == rule2.context.context
2429 && rule1.context.popCount == rule2.context.popCount;
2433 switch (rule1.type) {
2435 case Context::Rule::Type::AnyChar:
2436 case Context::Rule::Type::DetectChar:
2437 if ((rule2.type == Context::Rule::Type::AnyChar || rule2.type == Context::Rule::Type::DetectChar) && isCommonCompatible()
2438 && rule1.column == rule2.column) {
2439 qWarning() << filename <<
"line" << rule2.line <<
"can be merged as AnyChar with the previous rule";
2445 case Context::Rule::Type::RegExpr:
2446 if (rule2.type == Context::Rule::Type::RegExpr && isCommonCompatible() && rule1.dynamic == rule2.dynamic
2447 && (rule1.column == rule2.column || (rule1.column <= 0 && rule2.column <= 0))) {
2448 qWarning() << filename <<
"line" << rule2.line <<
"can be merged with the previous rule";
2453 case Context::Rule::Type::DetectSpaces:
2454 case Context::Rule::Type::HlCChar:
2455 case Context::Rule::Type::HlCHex:
2456 case Context::Rule::Type::HlCOct:
2457 case Context::Rule::Type::HlCStringChar:
2458 case Context::Rule::Type::Int:
2459 case Context::Rule::Type::Float:
2460 case Context::Rule::Type::LineContinue:
2461 case Context::Rule::Type::WordDetect:
2462 case Context::Rule::Type::StringDetect:
2463 case Context::Rule::Type::Detect2Chars:
2464 case Context::Rule::Type::IncludeRules:
2465 case Context::Rule::Type::DetectIdentifier:
2466 case Context::Rule::Type::keyword:
2467 case Context::Rule::Type::Unknown:
2468 case Context::Rule::Type::RangeDetect:
2484 void resolveContextName(Definition &definition, Context &context, ContextName &contextName,
int line)
2488 contextName.stay =
true;
2491 contextName.stay =
true;
2492 contextName.context = &context;
2494 qWarning() << definition.filename <<
"line" << line <<
"invalid context in" << context.name;
2500 ++contextName.popCount;
2507 qWarning() << definition.filename <<
"line" << line <<
"'!' missing between '#pop' and context name" << context.name;
2513 const int idx =
name.
indexOf(QStringLiteral(
"##"));
2515 auto it = definition.contexts.find(
name.toString());
2516 if (it != definition.contexts.end()) {
2517 contextName.context = &*it;
2520 auto defName =
name.
mid(idx + 2);
2521 auto it = m_definitions.find(defName.toString());
2522 if (it != m_definitions.end()) {
2523 auto listName =
name.
left(idx).toString();
2524 definition.referencedDefinitions.insert(&*it);
2525 auto ctxIt = it->contexts.find(listName.isEmpty() ? it->firstContextName : listName);
2526 if (ctxIt != it->contexts.end()) {
2527 contextName.context = &*ctxIt;
2530 qWarning() << definition.filename <<
"line" << line <<
"unknown definition in" << context.name;
2535 if (!contextName.context) {
2536 qWarning() << definition.filename <<
"line" << line <<
"unknown context" <<
name <<
"in" << context.name;
2544 Definition *m_currentDefinition =
nullptr;
2545 Keywords *m_currentKeywords =
nullptr;
2546 Context *m_currentContext =
nullptr;
2547 bool m_success =
true;
2554 QFile file(fileName);
2561 while (!xml.
atEnd()) {
2571 qWarning() <<
"XML error while reading" << fileName <<
" - " << qPrintable(xml.
errorString()) <<
"@ offset" << xml.
characterOffset();
2589 if (extensionParts.
isEmpty()) {
2594 for (
const auto &extension : extensionParts) {
2595 for (
const auto c : extension) {
2611 qWarning() <<
"invalid character" << c <<
"seen in extensions wildcard";
2622 int main(
int argc,
char *argv[])
2628 if (app.arguments().size() < 3) {
2632 #ifdef QT_XMLPATTERNS_LIB
2640 const QString hlFilenamesListing = app.arguments().value(3);
2641 if (hlFilenamesListing.
isEmpty()) {
2645 QStringList hlFilenames = readListing(hlFilenamesListing);
2647 qWarning(
"Failed to read %s", qPrintable(hlFilenamesListing));
2652 const QStringList textAttributes =
QStringList() << QStringLiteral(
"name") << QStringLiteral(
"section") << QStringLiteral(
"mimetype")
2653 << QStringLiteral(
"extensions") << QStringLiteral(
"style") << QStringLiteral(
"author")
2654 << QStringLiteral(
"license") << QStringLiteral(
"indenter");
2657 HlFilesChecker filesChecker;
2660 for (
const QString &hlFilename : std::as_const(hlFilenames)) {
2661 QFile hlFile(hlFilename);
2663 qWarning(
"Failed to open %s", qPrintable(hlFilename));
2668 #ifdef QT_XMLPATTERNS_LIB
2694 for (
const QString &attribute : std::as_const(textAttributes)) {
2699 if (!checkExtensions(hl[QStringLiteral(
"extensions")].
toString())) {
2700 qWarning() << hlFilename <<
"'extensions' wildcards invalid:" << hl[QStringLiteral(
"extensions")].toString();
2714 const QString hlName = hl[QStringLiteral(
"name")].toString();
2716 filesChecker.setDefinition(xml.
attributes().
value(QStringLiteral(
"kateversion")), hlFilename, hlName);
2719 while (!xml.
atEnd()) {
2721 filesChecker.processElement(xml);
2730 filesChecker.resolveContexts();
2732 if (!filesChecker.check()) {
2742 QFile outFile(app.arguments().at(1));