KItinerary

extractordocumentnodefactory.cpp
1/*
2 SPDX-FileCopyrightText: 2021 Volker Krause <vkrause@kde.org>
3
4 SPDX-License-Identifier: LGPL-2.0-or-later
5*/
6
7#include "extractordocumentnodefactory.h"
8#include "extractordocumentnode.h"
9#include "extractordocumentprocessor.h"
10#include "logging.h"
11
12#include "processors/binarydocumentprocessor.h"
13#include "processors/eradocumentprocessor.h"
14#include "processors/externalprocessor.h"
15#include "processors/htmldocumentprocessor.h"
16#include "processors/httpresponseprocessor.h"
17#include "processors/iatabcbpdocumentprocessor.h"
18#include "processors/icaldocumentprocessor.h"
19#include "processors/imagedocumentprocessor.h"
20#include "processors/jsonlddocumentprocessor.h"
21#include "processors/mimedocumentprocessor.h"
22#include "processors/pdfdocumentprocessor.h"
23#include "processors/pkpassdocumentprocessor.h"
24#include "processors/plistdocumentprocessor.h"
25#include "processors/textdocumentprocessor.h"
26#include "processors/uic9183documentprocessor.h"
27#include "processors/vdvdocumentprocessor.h"
28
29#include <QHash>
30#include <QMimeDatabase>
31
32using namespace KItinerary;
33
34enum {
35 MinDocumentSize = 4,
36 MaxDocumentSize = 10000000,
37};
38
39namespace KItinerary {
40class ExtractorDocumentNodeFactoryStatic {
41public:
42 ExtractorDocumentNodeFactoryStatic();
43
44 void registerProcessor(std::unique_ptr<ExtractorDocumentProcessor> &&processor, QStringView canonicalMimeType,
45 std::initializer_list<QStringView> aliasMimeTypes = {}, QStringView fallbackMimeType = {});
46
47 template <typename T>
48 inline void registerProcessor(QStringView canonicalMimeType, std::initializer_list<QStringView> aliasMimeTypes = {}, QStringView fallbackMimeType = {})
49 {
50 registerProcessor(std::make_unique<T>(), canonicalMimeType, aliasMimeTypes, fallbackMimeType);
51 }
52
53 void registerBuiltIn();
54 QStringView resolveAlias(QStringView mimeType) const;
55
56 struct ProcessorData {
57 QString mimeType;
58 const ExtractorDocumentProcessor* processor;
59 };
60 std::vector<ProcessorData> m_probeProcessors;
61 std::vector<ProcessorData> m_fallbackProbeProcessors;
62 std::vector<ProcessorData> m_mimetypeProcessorMap;
63 QHash<QString, QString> m_aliasMap;
64
65 // just for memory management
66 std::vector<std::unique_ptr<ExtractorDocumentProcessor>> processorPool;
67
68 static void insertProcessor(const ExtractorDocumentProcessor *proc, QStringView mimeType, std::vector<ProcessorData> &procMap);
69};
70
71class ExtractorDocumentNodeFactoryPrivate {
72public:
73 ExtractorDocumentNodeFactoryStatic *s;
74 std::unique_ptr<ExtractorDocumentProcessor> interceptProcessor;
75};
76}
77
78ExtractorDocumentNodeFactoryStatic::ExtractorDocumentNodeFactoryStatic()
79{
80 registerBuiltIn();
81}
82
83void ExtractorDocumentNodeFactoryStatic::insertProcessor(const ExtractorDocumentProcessor *proc, QStringView mimeType, std::vector<ProcessorData> &procMap)
84{
85 if (mimeType.empty()) {
86 return;
87 }
88
89 const auto it = std::lower_bound(procMap.begin(), procMap.end(), mimeType, [](const auto &proc, auto mt) {
90 return proc.mimeType < mt;
91 });
92 if (it != procMap.end() && (*it).mimeType == mimeType) {
93 qCWarning(Log) << "Document processor already registered for mimetype:" << mimeType;
94 return;
95 }
96
97 procMap.insert(it, { mimeType.toString(), proc });
98}
99
100void ExtractorDocumentNodeFactoryStatic::registerProcessor(std::unique_ptr<ExtractorDocumentProcessor> &&processor, QStringView canonicalMimeType,
101 std::initializer_list<QStringView> aliasMimeTypes, QStringView fallbackMimeType)
102{
103 insertProcessor(processor.get(), canonicalMimeType, m_probeProcessors);
104 insertProcessor(processor.get(), canonicalMimeType, m_mimetypeProcessorMap);
105 for (const auto mt : aliasMimeTypes) {
106 m_aliasMap.insert(mt.toString(), canonicalMimeType.isEmpty() ? fallbackMimeType.toString() : canonicalMimeType.toString());
107 }
108 if (!fallbackMimeType.isEmpty()) { // priorioty order matters for fallbacks, don't sort them!
109 m_fallbackProbeProcessors.push_back({ fallbackMimeType.toString(), processor.get() });
110 }
111 insertProcessor(processor.get(), fallbackMimeType, m_mimetypeProcessorMap);
112 processorPool.push_back(std::move(processor));
113}
114
115void ExtractorDocumentNodeFactoryStatic::registerBuiltIn()
116{
117 registerProcessor<PdfDocumentProcessor>(u"application/pdf");
118 registerProcessor<PkPassDocumentProcessor>(u"application/vnd.apple.pkpass");
119 registerProcessor<IcalEventProcessor>(u"internal/event");
120 registerProcessor<ImageDocumentProcessor>(u"internal/qimage", {u"image/png", u"image/jpeg", u"image/gif"});
121 registerProcessor<ElbDocumentProcessor>(u"internal/era-elb");
122 registerProcessor<SsbDocumentProcessor>(u"internal/era-ssb");
123 registerProcessor<IataBcbpDocumentProcessor>(u"internal/iata-bcbp");
124 registerProcessor<Uic9183DocumentProcessor>(u"internal/uic9183");
125 registerProcessor<VdvDocumentProcessor>(u"internal/vdv");
126 registerProcessor<IcalCalendarProcessor>(u"text/calendar");
127 registerProcessor<PListDocumentProcessor>(u"application/x-plist");
128 registerProcessor<HttpResponseProcessor>(u"internal/http-response");
129 registerProcessor<HarDocumentProcessor>(u"internal/har-archive");
130
131 // fallback types that catch a very broad set of input types
132 // order matters particularly here, the broadest ones need to go last
133 registerProcessor<JsonLdDocumentProcessor>({}, {u"application/json"}, u"application/ld+json");
134 registerProcessor<MimeDocumentProcessor>({}, {u"application/mbox"}, u"message/rfc822");
135 registerProcessor<HtmlDocumentProcessor>({}, {u"application/xhtml+xml"}, u"text/html");
136 registerProcessor<TextDocumentProcessor>({}, {}, u"text/plain");
137 registerProcessor<BinaryDocumentProcessor>({}, {}, u"application/octet-stream");
138}
139
140QStringView ExtractorDocumentNodeFactoryStatic::resolveAlias(QStringView mimeType) const
141{
142 const auto it = m_aliasMap.find(mimeType.toString());
143 if (it != m_aliasMap.end()) {
144 return it.value();
145 }
146 return mimeType;
147}
148
149
150ExtractorDocumentNodeFactory::ExtractorDocumentNodeFactory()
151 : d(std::make_unique<ExtractorDocumentNodeFactoryPrivate>())
152{
153 static ExtractorDocumentNodeFactoryStatic s_factory;
154 d->s = &s_factory;
155}
156
157ExtractorDocumentNodeFactory::~ExtractorDocumentNodeFactory() = default;
158
160{
161 if (data.size() <= MinDocumentSize || data.size() > MaxDocumentSize) {
162 return {};
163 }
164
165 if (d->interceptProcessor && d->interceptProcessor->canHandleData(data, fileName)) {
166 auto node = d->interceptProcessor->createNodeFromData(data);
167 if (node.mimeType().isEmpty()) {
168 node.setMimeType(QStringLiteral("internal/external-process"));
169 }
170 node.setProcessor(d->interceptProcessor.get());
171 return node;
172 }
173
174 if (mimeType.isEmpty()) {
175 QMimeDatabase db;
176 QString autoDetectedMimeType;
177 if (fileName.isEmpty()) {
178 autoDetectedMimeType = db.mimeTypeForData(data).name();
179 } else {
180 autoDetectedMimeType = db.mimeTypeForFileNameAndData(fileName.toString(), data).name();
181 }
182 mimeType = d->s->resolveAlias(autoDetectedMimeType);
183
184 // let processors check themselves if they support this data, or whether the auto-detected mimetype matches
185 for (const auto &p : d->s->m_probeProcessors) {
186 if (p.processor->canHandleData(data, fileName) || (!mimeType.isEmpty() && p.mimeType == mimeType)) {
187 auto node = p.processor->createNodeFromData(data);
188 if (node.content().isNull()) {
189 continue;
190 }
191
192 node.setMimeType(p.mimeType);
193 node.setProcessor(p.processor);
194 return node;
195 }
196 }
197
198 // try the basic types that ultimately will accept anything
199 for (const auto &p : d->s->m_fallbackProbeProcessors) {
200 if (p.processor->canHandleData(data, fileName)) {
201 auto node = p.processor->createNodeFromData(data);
202 if (node.content().isNull()) {
203 continue;
204 }
205
206 node.setMimeType(p.mimeType);
207 node.setProcessor(p.processor);
208 return node;
209 }
210 }
211
212 // unreachable as application/octet-stream will always match
213 return {};
214 }
215
216 mimeType = d->s->resolveAlias(mimeType);
217 const auto it = std::lower_bound(d->s->m_mimetypeProcessorMap.begin(), d->s->m_mimetypeProcessorMap.end(), mimeType, [](const auto &proc, auto mt) {
218 return proc.mimeType < mt;
219 });
220 if (it == d->s->m_mimetypeProcessorMap.end() || (*it).mimeType != mimeType) {
221 qCDebug(Log) << "No document processor found for mimetype" << mimeType;
222 return {};
223 }
224
225 auto node = (*it).processor->createNodeFromData(data);
226 node.setMimeType((*it).mimeType);
227 node.setProcessor((*it).processor);
228 return node;
229}
230
232{
233 mimeType = d->s->resolveAlias(mimeType);
234 const auto it = std::lower_bound(d->s->m_mimetypeProcessorMap.begin(), d->s->m_mimetypeProcessorMap.end(), mimeType, [](const auto &proc, auto mt) {
235 return proc.mimeType < mt;
236 });
237 if (it == d->s->m_mimetypeProcessorMap.end() || (*it).mimeType != mimeType) {
238 qCDebug(Log) << "No document processor found for mimetype" << mimeType;
239 return {};
240 }
241
242 auto node = (*it).processor->createNodeFromContent(decodedData);
243 node.setMimeType((*it).mimeType);
244 node.setProcessor((*it).processor);
245 return node;
246}
247
248void ExtractorDocumentNodeFactory::registerProcessor(std::unique_ptr<ExtractorDocumentProcessor> &&processor, QStringView mimeType,
249 std::initializer_list<QStringView> aliasMimeTypes)
250{
251 d->s->registerProcessor(std::move(processor), mimeType, aliasMimeTypes);
252}
253
255{
256 if (separateProcess && !d->interceptProcessor) {
257 d->interceptProcessor = std::make_unique<ExternalProcessor>();
258 } else if (!separateProcess && d->interceptProcessor) {
259 d->interceptProcessor.reset();
260 }
261}
void registerProcessor(std::unique_ptr< ExtractorDocumentProcessor > &&processor, QStringView canonicalMimeType, std::initializer_list< QStringView > aliasMimeTypes={})
Register a new document processor.
ExtractorDocumentNode createNode(const QByteArray &data, QStringView fileName={}, QStringView mimeType={}) const
Create a new document node from data.
void setUseSeparateProcess(bool separateProcess)
Perform extraction of "risky" content such as PDF files in a separate process.
A node in the extracted document object tree.
Abstract base class of a document type processor.
char * toString(const EngineQuery &query)
KCALUTILS_EXPORT QString mimeType()
Classes for reservation/travel data models, data extraction and data augmentation.
Definition berelement.h:17
qsizetype size() const const
iterator end()
iterator find(const Key &key)
iterator insert(const Key &key, const T &value)
QMimeType mimeTypeForData(QIODevice *device) const const
QMimeType mimeTypeForFileNameAndData(const QString &fileName, QIODevice *device) const const
QString & insert(qsizetype position, QChar ch)
bool isEmpty() const const
bool isEmpty() const const
QString toString() const const
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Fri Jun 7 2024 11:52:32 by doxygen 1.10.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.