KFileMetaData

externalextractor.cpp
1 /*
2  * This file is part of the KFileMetaData project
3  * Copyright (C) 2016 Varun Joshi <[email protected]>
4  * Copyright (C) 2015 Boudhayan Gupta <[email protected]>
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) version 3, or any
10  * later version accepted by the membership of KDE e.V. (or its
11  * successor approved by the membership of KDE e.V.), which shall
12  * act as a proxy defined in Section 6 of version 3 of the license.
13  *
14  * This library is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17  * Lesser General Public License for more details.
18  *
19  * You should have received a copy of the GNU Lesser General Public
20  * License along with this library. If not, see <http://www.gnu.org/licenses/>.
21  *
22  */
23 
24 #include "externalextractor.h"
25 #include "kfilemetadata_debug.h"
26 #include "properties.h"
27 #include "propertyinfo.h"
28 #include "typeinfo.h"
29 
30 #include <QDir>
31 #include <QProcess>
32 #include <QJsonDocument>
33 #include <QJsonObject>
34 #include <QJsonArray>
35 
36 #define EXTRACTOR_TIMEOUT_MS 30000
37 
38 using namespace KFileMetaData;
39 
40 class Q_DECL_HIDDEN ExternalExtractor::ExternalExtractorPrivate
41 {
42 public:
43  QString path;
44  QStringList writeMimetypes;
45  QString mainPath;
46 };
47 
48 
49 ExternalExtractor::ExternalExtractor(QObject* parent)
50  : ExtractorPlugin(parent),
51  d_ptr(new ExternalExtractorPrivate)
52 {
53 }
54 
55 ExternalExtractor::ExternalExtractor(const QString& pluginPath)
56  : ExtractorPlugin(nullptr),
57  d_ptr(new ExternalExtractorPrivate)
58 {
59  Q_D(ExternalExtractor);
60 
61  d->path = pluginPath;
62 
63  QDir pluginDir(pluginPath);
64  QStringList pluginDirContents = pluginDir.entryList(QDir::Files | QDir::NoDotAndDotDot);
65 
66  if (!pluginDirContents.contains(QStringLiteral("manifest.json"))) {
67  qCDebug(KFILEMETADATA_LOG) << pluginPath << "does not seem to contain a valid plugin";
68  return;
69  }
70 
71  QFile manifest(pluginDir.filePath(QStringLiteral("manifest.json")));
72  manifest.open(QIODevice::ReadOnly);
73  QJsonDocument manifestDoc = QJsonDocument::fromJson(manifest.readAll());
74  if (!manifestDoc.isObject()) {
75  qCDebug(KFILEMETADATA_LOG) << "Manifest does not seem to be a valid JSON Object";
76  return;
77  }
78 
79  QJsonObject rootObject = manifestDoc.object();
80  const QJsonArray mimetypesArray = rootObject.value(QStringLiteral("mimetypes")).toArray();
81  QStringList mimetypes;
82  mimetypes.reserve(mimetypesArray.count());
83 
84  for (const QJsonValue &mimetype : mimetypesArray) {
85  mimetypes << mimetype.toString();
86  }
87 
88  d->writeMimetypes.append(mimetypes);
89  d->mainPath = pluginDir.filePath(rootObject[QStringLiteral("main")].toString());
90 }
91 
92 ExternalExtractor::~ExternalExtractor()
93 {
94  delete d_ptr;
95 }
96 
97 QStringList ExternalExtractor::mimetypes() const
98 {
99  Q_D(const ExternalExtractor);
100 
101  return d->writeMimetypes;
102 }
103 
104 void ExternalExtractor::extract(ExtractionResult* result)
105 {
106  Q_D(ExternalExtractor);
107 
108  QJsonDocument writeData;
109  QJsonObject writeRootObject;
110  QByteArray output;
111  QByteArray errorOutput;
112 
113  writeRootObject[QStringLiteral("path")] = QJsonValue(result->inputUrl());
114  writeRootObject[QStringLiteral("mimetype")] = result->inputMimetype();
115  writeData.setObject(writeRootObject);
116 
117  QProcess extractorProcess;
118  extractorProcess.start(d->mainPath, QStringList(), QIODevice::ReadWrite);
119  bool started = extractorProcess.waitForStarted();
120  if (!started) {
121  qCWarning(KFILEMETADATA_LOG) << "External extractor" << d->mainPath
122  << "failed to start:" << extractorProcess.errorString();
123  return;
124  }
125 
126  extractorProcess.write(writeData.toJson());
127  extractorProcess.closeWriteChannel();
128  extractorProcess.waitForFinished(EXTRACTOR_TIMEOUT_MS);
129 
130  output = extractorProcess.readAll();
131  errorOutput = extractorProcess.readAllStandardError();
132 
133  if (extractorProcess.exitStatus()) {
134  qCWarning(KFILEMETADATA_LOG) << "External extractor" << d->mainPath
135  << "failed to index" << result->inputUrl() << "-" << errorOutput;
136  return;
137  }
138 
139  // now we read in the output (which is a standard json format) into the
140  // ExtractionResult
141 
142  QJsonDocument extractorData = QJsonDocument::fromJson(output);
143  if (!extractorData.isObject()) {
144  return;
145  }
146  QJsonObject rootObject = extractorData.object();
147  QJsonObject propertiesObject = rootObject[QStringLiteral("properties")].toObject();
148 
149  const auto propertiesObjectEnd = propertiesObject.constEnd();
150  auto i = propertiesObject.constBegin();
151  for (; i != propertiesObjectEnd; ++i) {
152  if (i.key() == QStringLiteral("typeInfo")) {
153  TypeInfo info = TypeInfo::fromName(propertiesObject.value(i.key()).toString());
154  result->addType(info.type());
155  continue;
156  }
157 
158  // for plaintext extraction
159  if (i.key() == QStringLiteral("text")) {
160  result->append(propertiesObject.value(i.key()).toString(QStringLiteral("")));
161  continue;
162  }
163 
164  PropertyInfo info = PropertyInfo::fromName(i.key());
165  if (info.name() != i.key()) {
166  continue;
167  }
168  result->add(info.property(), i.value().toVariant());
169  }
170 
171  if (rootObject[QStringLiteral("status")].toString() != QStringLiteral("OK")) {
172  qCDebug(KFILEMETADATA_LOG) << rootObject[QStringLiteral("error")].toString();
173  }
174 }
QJsonDocument fromJson(const QByteArray &json, QJsonParseError *error)
The PropertyInfo class can be used to obtain extra information about any property.
Definition: propertyinfo.h:41
QByteArray toJson() const const
virtual void append(const QString &text)=0
This function is called by plugins when they wish for some plain text to be indexed without any prope...
QString errorString() const const
QJsonObject::const_iterator constEnd() const const
void reserve(int alloc)
virtual void add(Property::Property property, const QVariant &value)=0
This function is called by the plugins when they wish to add a key value pair which should be indexed...
QJsonObject object() const const
The ExtractorPlugin is the base class for all file metadata extractors.
bool contains(const QString &str, Qt::CaseSensitivity cs) const const
bool isObject() const const
int count() const const
QJsonObject::const_iterator constBegin() const const
virtual void addType(Type::Type type)=0
This function is called by the plugins.
static PropertyInfo fromName(const QString &name)
Construct a PropertyInfo from the internal property name.
QJsonArray toArray() const const
QByteArray readAll()
KIOCORE_EXPORT MimetypeJob * mimetype(const QUrl &url, JobFlags flags=DefaultFlags)
QString inputMimetype() const
The input mimetype.
virtual bool open(QIODevice::OpenMode mode) override
bool waitForStarted(int msecs)
<KFileMetaData/Properties>
char * toString(const T &value)
Property::Property property() const
The enumeration which represents this property.
qint64 write(const char *data, qint64 maxSize)
void setObject(const QJsonObject &object)
QJsonValue value(const QString &key) const const
QString name() const
The internal unique name used to refer to the property.
void closeWriteChannel()
The ExtractionResult class is where all the data extracted by the indexer is saved.
QProcess::ExitStatus exitStatus() const const
void start(const QString &program, const QStringList &arguments, QIODevice::OpenMode mode)
QByteArray readAllStandardError()
QString inputUrl() const
The input url which the plugins will use to locate the file.
bool waitForFinished(int msecs)
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Tue May 26 2020 23:11:36 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.