7#include "extractordocumentnodefactory.h"
8#include "extractordocumentnode.h"
9#include "extractordocumentprocessor.h"
12#include "processors/binarydocumentprocessor.h"
13#include "processors/eradocumentprocessor.h"
14#include "processors/externalprocessor.h"
15#include "processors/htmldocumentprocessor.h"
16#include "processors/httpresponseprocessor.h"
17#include "processors/iatabcbpdocumentprocessor.h"
18#include "processors/icaldocumentprocessor.h"
19#include "processors/imagedocumentprocessor.h"
20#include "processors/jsonlddocumentprocessor.h"
21#include "processors/mimedocumentprocessor.h"
22#include "processors/pdfdocumentprocessor.h"
23#include "processors/pkpassdocumentprocessor.h"
24#include "processors/plistdocumentprocessor.h"
25#include "processors/textdocumentprocessor.h"
26#include "processors/uic9183documentprocessor.h"
27#include "processors/vdvdocumentprocessor.h"
30#include <QMimeDatabase>
36 MaxDocumentSize = 10000000,
40class ExtractorDocumentNodeFactoryStatic {
42 ExtractorDocumentNodeFactoryStatic();
44 void registerProcessor(std::unique_ptr<ExtractorDocumentProcessor> &&processor,
QStringView canonicalMimeType,
45 std::initializer_list<QStringView> aliasMimeTypes = {},
QStringView fallbackMimeType = {});
48 inline void registerProcessor(
QStringView canonicalMimeType, std::initializer_list<QStringView> aliasMimeTypes = {},
QStringView fallbackMimeType = {})
50 registerProcessor(std::make_unique<T>(), canonicalMimeType, aliasMimeTypes, fallbackMimeType);
53 void registerBuiltIn();
56 struct ProcessorData {
60 std::vector<ProcessorData> m_probeProcessors;
61 std::vector<ProcessorData> m_fallbackProbeProcessors;
62 std::vector<ProcessorData> m_mimetypeProcessorMap;
66 std::vector<std::unique_ptr<ExtractorDocumentProcessor>> processorPool;
71class ExtractorDocumentNodeFactoryPrivate {
73 ExtractorDocumentNodeFactoryStatic *s;
74 std::unique_ptr<ExtractorDocumentProcessor> interceptProcessor;
78ExtractorDocumentNodeFactoryStatic::ExtractorDocumentNodeFactoryStatic()
89 const auto it = std::lower_bound(procMap.begin(), procMap.end(), mimeType, [](
const auto &proc,
auto mt) {
90 return proc.mimeType < mt;
92 if (it != procMap.end() && (*it).mimeType == mimeType) {
93 qCWarning(
Log) <<
"Document processor already registered for mimetype:" <<
mimeType;
100void ExtractorDocumentNodeFactoryStatic::registerProcessor(std::unique_ptr<ExtractorDocumentProcessor> &&processor,
QStringView canonicalMimeType,
101 std::initializer_list<QStringView> aliasMimeTypes,
QStringView fallbackMimeType)
103 insertProcessor(processor.get(), canonicalMimeType, m_probeProcessors);
104 insertProcessor(processor.get(), canonicalMimeType, m_mimetypeProcessorMap);
105 for (
const auto mt : aliasMimeTypes) {
108 insertProcessor(processor.get(), fallbackMimeType, m_fallbackProbeProcessors);
109 insertProcessor(processor.get(), fallbackMimeType, m_mimetypeProcessorMap);
110 processorPool.push_back(std::move(processor));
113void ExtractorDocumentNodeFactoryStatic::registerBuiltIn()
115 registerProcessor<PdfDocumentProcessor>(u
"application/pdf");
116 registerProcessor<PkPassDocumentProcessor>(u
"application/vnd.apple.pkpass");
117 registerProcessor<IcalEventProcessor>(u
"internal/event");
118 registerProcessor<ImageDocumentProcessor>(u
"internal/qimage", {u
"image/png", u
"image/jpeg", u
"image/gif"});
119 registerProcessor<ElbDocumentProcessor>(u
"internal/era-elb");
120 registerProcessor<SsbDocumentProcessor>(u
"internal/era-ssb");
121 registerProcessor<IataBcbpDocumentProcessor>(u
"internal/iata-bcbp");
122 registerProcessor<Uic9183DocumentProcessor>(u
"internal/uic9183");
123 registerProcessor<VdvDocumentProcessor>(u
"internal/vdv");
124 registerProcessor<IcalCalendarProcessor>(u
"text/calendar");
125 registerProcessor<PListDocumentProcessor>(u
"application/x-plist");
126 registerProcessor<HttpResponseProcessor>(u
"internal/http-response");
127 registerProcessor<HarDocumentProcessor>(u
"internal/har-archive");
131 registerProcessor<JsonLdDocumentProcessor>({}, {u
"application/json"}, u
"application/ld+json");
132 registerProcessor<MimeDocumentProcessor>({}, {u
"application/mbox"}, u
"message/rfc822");
133 registerProcessor<HtmlDocumentProcessor>({}, {u
"application/xhtml+xml"}, u
"text/html");
134 registerProcessor<TextDocumentProcessor>({}, {}, u
"text/plain");
135 registerProcessor<BinaryDocumentProcessor>({}, {}, u
"application/octet-stream");
141 if (it != m_aliasMap.
end()) {
148ExtractorDocumentNodeFactory::ExtractorDocumentNodeFactory()
149 : d(
std::make_unique<ExtractorDocumentNodeFactoryPrivate>())
151 static ExtractorDocumentNodeFactoryStatic s_factory;
155ExtractorDocumentNodeFactory::~ExtractorDocumentNodeFactory() =
default;
159 if (data.
size() <= MinDocumentSize || data.
size() > MaxDocumentSize) {
163 if (d->interceptProcessor && d->interceptProcessor->canHandleData(data, fileName)) {
164 auto node = d->interceptProcessor->createNodeFromData(data);
165 if (node.mimeType().isEmpty()) {
166 node.setMimeType(QStringLiteral(
"internal/external-process"));
168 node.setProcessor(d->interceptProcessor.get());
175 for (
const auto &p : d->s->m_probeProcessors) {
176 if (p.processor->canHandleData(data, fileName)) {
177 auto node = p.processor->createNodeFromData(data);
178 if (node.content().isNull()) {
182 node.setMimeType(p.mimeType);
183 node.setProcessor(p.processor);
188 for (
const auto &p : d->s->m_fallbackProbeProcessors) {
189 if (p.processor->canHandleData(data, fileName)) {
190 auto node = p.processor->createNodeFromData(data);
191 if (node.content().isNull()) {
195 node.setMimeType(p.mimeType);
196 node.setProcessor(p.processor);
208 mimeType = autoDetectedMimeType;
211 mimeType = d->s->resolveAlias(mimeType);
212 const auto it = std::lower_bound(d->s->m_mimetypeProcessorMap.begin(), d->s->m_mimetypeProcessorMap.end(), mimeType, [](
const auto &proc,
auto mt) {
213 return proc.mimeType < mt;
215 if (it == d->s->m_mimetypeProcessorMap.end() || (*it).mimeType != mimeType) {
216 qCDebug(
Log) <<
"No document processor found for mimetype" << mimeType;
220 auto node = (*it).processor->createNodeFromData(data);
221 node.setMimeType((*it).mimeType);
222 node.setProcessor((*it).processor);
228 mimeType = d->s->resolveAlias(mimeType);
229 const auto it = std::lower_bound(d->s->m_mimetypeProcessorMap.begin(), d->s->m_mimetypeProcessorMap.end(), mimeType, [](
const auto &proc,
auto mt) {
230 return proc.mimeType < mt;
232 if (it == d->s->m_mimetypeProcessorMap.end() || (*it).mimeType != mimeType) {
233 qCDebug(
Log) <<
"No document processor found for mimetype" << mimeType;
237 auto node = (*it).processor->createNodeFromContent(decodedData);
238 node.setMimeType((*it).mimeType);
239 node.setProcessor((*it).processor);
244 std::initializer_list<QStringView> aliasMimeTypes)
246 d->s->registerProcessor(std::move(processor), mimeType, aliasMimeTypes);
251 if (separateProcess && !d->interceptProcessor) {
252 d->interceptProcessor = std::make_unique<ExternalProcessor>();
253 }
else if (!separateProcess && d->interceptProcessor) {
254 d->interceptProcessor.reset();
char * toString(const EngineQuery &query)
KCALUTILS_EXPORT QString mimeType()
Classes for reservation/travel data models, data extraction and data augmentation.
qsizetype size() const const
iterator find(const Key &key)
iterator insert(const Key &key, const T &value)
QMimeType mimeTypeForData(QIODevice *device) const const
QMimeType mimeTypeForFileNameAndData(const QString &fileName, QIODevice *device) const const
QString & insert(qsizetype position, QChar ch)
bool isEmpty() const const
bool isEmpty() const const
QString toString() const const