Messagelib

createphishingurldatabasejob.cpp
1 /*
2  SPDX-FileCopyrightText: 2016-2021 Laurent Montel <[email protected]>
3 
4  SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6 
7 #include "createphishingurldatabasejob.h"
8 #include "checkphishingurlutil.h"
9 #include "updatedatabaseinfo.h"
10 #include "webengineviewer_debug.h"
11 #include <PimCommon/NetworkManager>
12 
13 #include <QJsonDocument>
14 #include <QNetworkConfigurationManager>
15 #include <QUrlQuery>
16 
17 using namespace WebEngineViewer;
18 
19 WEBENGINEVIEWER_EXPORT bool webengineview_useCompactJson_CreatePhishingUrlDataBaseJob = true;
20 
21 class WebEngineViewer::CreatePhishingUrlDataBaseJobPrivate
22 {
23 public:
24  Q_REQUIRED_RESULT UpdateDataBaseInfo::CompressionType parseCompressionType(const QString &str);
25  Q_REQUIRED_RESULT RiceDeltaEncoding parseRiceDeltaEncoding(const QMap<QString, QVariant> &map);
26  Q_REQUIRED_RESULT QVector<Removal> parseRemovals(const QVariantList &lst);
27  Q_REQUIRED_RESULT QVector<Addition> parseAdditions(const QVariantList &lst);
28  QString mDataBaseState;
29  CreatePhishingUrlDataBaseJob::ContraintsCompressionType mContraintsCompressionType = CreatePhishingUrlDataBaseJob::RawCompression;
30  CreatePhishingUrlDataBaseJob::DataBaseDownloadType mDataBaseDownloadNeeded = CreatePhishingUrlDataBaseJob::FullDataBase;
31  QNetworkAccessManager *mNetworkAccessManager = nullptr;
32 };
33 
34 CreatePhishingUrlDataBaseJob::CreatePhishingUrlDataBaseJob(QObject *parent)
35  : QObject(parent)
36  , d(new CreatePhishingUrlDataBaseJobPrivate)
37 {
38  d->mNetworkAccessManager = new QNetworkAccessManager(this);
39  d->mNetworkAccessManager->setRedirectPolicy(QNetworkRequest::NoLessSafeRedirectPolicy);
40  d->mNetworkAccessManager->setStrictTransportSecurityEnabled(true);
41  d->mNetworkAccessManager->enableStrictTransportSecurityStore(true);
42 
43  connect(d->mNetworkAccessManager, &QNetworkAccessManager::finished, this, &CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished);
44  connect(d->mNetworkAccessManager, &QNetworkAccessManager::sslErrors, this, &CreatePhishingUrlDataBaseJob::slotSslErrors);
45 }
46 
47 CreatePhishingUrlDataBaseJob::~CreatePhishingUrlDataBaseJob() = default;
48 
49 void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)
50 {
51  qCDebug(WEBENGINEVIEWER_LOG) << " void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)" << error.count();
52  reply->ignoreSslErrors(error);
53 }
54 
55 void CreatePhishingUrlDataBaseJob::start()
56 {
57  if (!PimCommon::NetworkManager::self()->networkConfigureManager()->isOnline()) {
58  Q_EMIT finished(UpdateDataBaseInfo(), BrokenNetwork);
59  deleteLater();
60  } else {
62  query.addQueryItem(QStringLiteral("key"), WebEngineViewer::CheckPhishingUrlUtil::apiKey());
63  QUrl safeUrl = QUrl(QStringLiteral("https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch"));
64  safeUrl.setQuery(query);
65  // qCDebug(WEBENGINEVIEWER_LOG) << " safeUrl" << safeUrl;
66  QNetworkRequest request(safeUrl);
67  request.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json"));
68 
69  const QByteArray baPostData = jsonRequest();
70  Q_EMIT debugJson(baPostData);
71  qCDebug(WEBENGINEVIEWER_LOG) << " postData.toJson()" << baPostData;
72  // curl -H "Content-Type: application/json" -X POST -d
73  // '{"client":{"clientId":"KDE","clientVersion":"5.4.0"},"threatInfo":{"platformTypes":["WINDOWS"],"threatEntries":[{"url":"http://www.kde.org"}],"threatEntryTypes":["URL"],"threatTypes":["MALWARE"]}}'
74  // https://safebrowsing.googleapis.com/v4/threatMatches:find?key=AIzaSyBS62pXATjabbH2RM_jO2EzDg1mTMHlnyo
75  QNetworkReply *reply = d->mNetworkAccessManager->post(request, baPostData);
76  connect(reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::errorOccurred), this, &CreatePhishingUrlDataBaseJob::slotError);
77  }
78 }
79 
80 void CreatePhishingUrlDataBaseJob::setDataBaseState(const QString &value)
81 {
82  d->mDataBaseState = value;
83 }
84 
85 void CreatePhishingUrlDataBaseJob::slotError(QNetworkReply::NetworkError error)
86 {
87  auto reply = qobject_cast<QNetworkReply *>(sender());
88  qWarning() << " error " << error << " error string : " << reply->errorString();
89  reply->deleteLater();
90  deleteLater();
91 }
92 
93 QByteArray CreatePhishingUrlDataBaseJob::jsonRequest() const
94 {
95 #if 0
96  {
97  "client" : {
98  "clientId" : "yourcompanyname",
99  "clientVersion" : "1.5.2"
100  },
101  "listUpdateRequests" : [{
102  "threatType" : "MALWARE",
103  "platformType" : "WINDOWS",
104  "threatEntryType" : "URL",
105  "state" : "Gg4IBBADIgYQgBAiAQEoAQ==",
106  "constraints" : {
107  "maxUpdateEntries" : 2048,
108  "maxDatabaseEntries" : 4096,
109  "region" : "US",
110  "supportedCompressions" : ["RAW"]
111  }
112  }]
113  }
114 #endif
115  QVariantMap clientMap;
116  QVariantMap map;
117 
118  clientMap.insert(QStringLiteral("clientId"), QStringLiteral("KDE"));
119  clientMap.insert(QStringLiteral("clientVersion"), CheckPhishingUrlUtil::versionApps());
120  map.insert(QStringLiteral("client"), clientMap);
121 
122  QVariantList listUpdateRequests;
123 
124  QVariantMap threatMap;
125  threatMap.insert(QStringLiteral("platformType"), QStringLiteral("WINDOWS"));
126  threatMap.insert(QStringLiteral("threatType"), QStringLiteral("MALWARE"));
127  threatMap.insert(QStringLiteral("threatEntryType"), QStringLiteral("URL"));
128 
129  // Contrainsts
130  QVariantMap contraintsMap;
131  QVariantList contraintsCompressionList;
132  QString compressionStr;
133  switch (d->mContraintsCompressionType) {
134  case RiceCompression:
135  compressionStr = QStringLiteral("RICE");
136  break;
137  case RawCompression:
138  compressionStr = QStringLiteral("RAW");
139  break;
140  }
141  contraintsCompressionList.append(compressionStr);
142  contraintsMap.insert(QStringLiteral("supportedCompressions"), contraintsCompressionList);
143  threatMap.insert(QStringLiteral("constraints"), contraintsMap);
144 
145  // Define state when we want to define update database. Empty is full.
146  switch (d->mDataBaseDownloadNeeded) {
147  case FullDataBase:
148  qCDebug(WEBENGINEVIEWER_LOG) << " full update";
149  threatMap.insert(QStringLiteral("state"), QString());
150  break;
151  case UpdateDataBase:
152  qCDebug(WEBENGINEVIEWER_LOG) << " update database";
153  if (d->mDataBaseState.isEmpty()) {
154  qCWarning(WEBENGINEVIEWER_LOG) << "Partial Download asked but database set is empty";
155  }
156  threatMap.insert(QStringLiteral("state"), d->mDataBaseState);
157  break;
158  }
159 
160  listUpdateRequests.append(threatMap);
161 
162  map.insert(QStringLiteral("listUpdateRequests"), listUpdateRequests);
163 
164  const QJsonDocument postData = QJsonDocument::fromVariant(map);
165  const QByteArray baPostData = postData.toJson(webengineview_useCompactJson_CreatePhishingUrlDataBaseJob ? QJsonDocument::Compact : QJsonDocument::Indented);
166  return baPostData;
167 }
168 
169 void CreatePhishingUrlDataBaseJob::setDataBaseDownloadNeeded(CreatePhishingUrlDataBaseJob::DataBaseDownloadType type)
170 {
171  d->mDataBaseDownloadNeeded = type;
172 }
173 
174 void CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished(QNetworkReply *reply)
175 {
176  const QByteArray returnValue(reply->readAll());
177  Q_EMIT debugJsonResult(returnValue);
178  parseResult(returnValue);
179  reply->deleteLater();
180 }
181 
182 RiceDeltaEncoding CreatePhishingUrlDataBaseJobPrivate::parseRiceDeltaEncoding(const QMap<QString, QVariant> &map)
183 {
184  RiceDeltaEncoding riceDeltaEncodingTmp;
185  QMap<QString, QVariant>::const_iterator riceHashesIt = map.cbegin();
186  const QMap<QString, QVariant>::const_iterator riceHashesItEnd = map.cend();
187  for (; riceHashesIt != riceHashesItEnd; ++riceHashesIt) {
188  const QString key = riceHashesIt.key();
189  if (key == QLatin1String("firstValue")) {
190  riceDeltaEncodingTmp.firstValue = riceHashesIt.value().toByteArray();
191  } else if (key == QLatin1String("riceParameter")) {
192  riceDeltaEncodingTmp.riceParameter = riceHashesIt.value().toInt();
193  } else if (key == QLatin1String("numEntries")) {
194  riceDeltaEncodingTmp.numberEntries = riceHashesIt.value().toInt();
195  } else if (key == QLatin1String("encodedData")) {
196  riceDeltaEncodingTmp.encodingData = riceHashesIt.value().toByteArray();
197  } else {
198  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRiceDeltaEncoding unknown riceDeltaEncoding key " << key;
199  }
200  }
201  return riceDeltaEncodingTmp;
202 }
203 
204 QVector<Addition> CreatePhishingUrlDataBaseJobPrivate::parseAdditions(const QVariantList &lst)
205 {
206  QVector<Addition> additionList;
207  for (const QVariant &v : lst) {
208  if (v.canConvert<QVariantMap>()) {
209  QMapIterator<QString, QVariant> mapIt(v.toMap());
210  Addition tmp;
211  while (mapIt.hasNext()) {
212  mapIt.next();
213  const QString keyStr = mapIt.key();
214  if (keyStr == QLatin1String("compressionType")) {
215  tmp.compressionType = parseCompressionType(mapIt.value().toString());
216  } else if (keyStr == QLatin1String("riceHashes")) {
217  RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
218  if (riceDeltaEncodingTmp.isValid()) {
219  tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
220  }
221  } else if (keyStr == QLatin1String("rawHashes")) {
222  QMapIterator<QString, QVariant> rawHashesIt(mapIt.value().toMap());
223  while (rawHashesIt.hasNext()) {
224  rawHashesIt.next();
225  const QString key = rawHashesIt.key();
226  if (key == QLatin1String("rawHashes")) {
227  tmp.hashString = QByteArray::fromBase64(rawHashesIt.value().toByteArray());
228  } else if (key == QLatin1String("prefixSize")) {
229  tmp.prefixSize = rawHashesIt.value().toInt();
230  } else {
231  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown rawHashes key " << key;
232  }
233  }
234  } else {
235  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown mapIt.key() " << keyStr;
236  }
237  }
238  if (tmp.isValid()) {
239  additionList.append(tmp);
240  }
241  } else {
242  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions not parsing type " << v.typeName();
243  }
244  }
245  return additionList;
246 }
247 
248 UpdateDataBaseInfo::CompressionType CreatePhishingUrlDataBaseJobPrivate::parseCompressionType(const QString &str)
249 {
250  UpdateDataBaseInfo::CompressionType type(UpdateDataBaseInfo::UnknownCompression);
251  if (str == QLatin1String("COMPRESSION_TYPE_UNSPECIFIED")) {
252  type = UpdateDataBaseInfo::UnknownCompression;
253  } else if (str == QLatin1String("RICE")) {
254  type = UpdateDataBaseInfo::RiceCompression;
255  } else if (str == QLatin1String("RAW")) {
256  type = UpdateDataBaseInfo::RawCompression;
257  } else {
258  qCWarning(WEBENGINEVIEWER_LOG) << "CreatePhishingUrlDataBaseJob::parseCompressionType unknown compression type " << str;
259  }
260  return type;
261 }
262 
263 QVector<Removal> CreatePhishingUrlDataBaseJobPrivate::parseRemovals(const QVariantList &lst)
264 {
265  QVector<Removal> removalList;
266  for (const QVariant &v : lst) {
267  if (v.canConvert<QVariantMap>()) {
268  Removal tmp;
269  QMapIterator<QString, QVariant> mapIt(v.toMap());
270  while (mapIt.hasNext()) {
271  mapIt.next();
272  const QString keyStr = mapIt.key();
273  if (keyStr == QLatin1String("compressionType")) {
274  tmp.compressionType = parseCompressionType(mapIt.value().toString());
275  } else if (keyStr == QLatin1String("riceIndices")) {
276  RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
277  if (riceDeltaEncodingTmp.isValid()) {
278  tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
279  }
280  } else if (keyStr == QLatin1String("rawIndices")) {
281  const QVariantMap map = mapIt.value().toMap();
282  QMapIterator<QString, QVariant> rawIndicesIt(map);
283  while (rawIndicesIt.hasNext()) {
284  rawIndicesIt.next();
285  if (rawIndicesIt.key() == QLatin1String("indices")) {
286  const QVariantList rawList = rawIndicesIt.value().toList();
287  QList<quint32> indexList;
288  indexList.reserve(rawList.count());
289  for (const QVariant &var : rawList) {
290  indexList.append(var.toUInt());
291  }
292  tmp.indexes = indexList;
293  } else {
294  qCDebug(WEBENGINEVIEWER_LOG) << "rawIndicesIt.key() unknown " << rawIndicesIt.key();
295  }
296  }
297  } else {
298  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals unknown mapIt.key() " << keyStr;
299  }
300  }
301  if (tmp.isValid()) {
302  removalList.append(tmp);
303  }
304  } else {
305  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals not parsing type " << v.typeName();
306  }
307  }
308  return removalList;
309 }
310 
311 void CreatePhishingUrlDataBaseJob::parseResult(const QByteArray &value)
312 {
313  UpdateDataBaseInfo databaseInfo;
314  QJsonDocument document = QJsonDocument::fromJson(value);
315  if (document.isNull()) {
316  Q_EMIT finished(databaseInfo, InvalidData);
317  } else {
318  const QVariantMap answer = document.toVariant().toMap();
319  if (answer.isEmpty()) {
320  Q_EMIT finished(databaseInfo, InvalidData);
321  } else {
323  while (i.hasNext()) {
324  i.next();
325  if (i.key() == QLatin1String("listUpdateResponses")) {
326  const QVariantList info = i.value().toList();
327  if (info.count() == 1) {
328  const QVariant infoVar = info.at(0);
329  if (infoVar.canConvert<QVariantMap>()) {
330  QMapIterator<QString, QVariant> mapIt(infoVar.toMap());
331  while (mapIt.hasNext()) {
332  mapIt.next();
333  const QString mapKey = mapIt.key();
334  if (mapKey == QLatin1String("additions")) {
335  const QVariantList lst = mapIt.value().toList();
336  const QVector<Addition> addList = d->parseAdditions(lst);
337  if (!addList.isEmpty()) {
338  databaseInfo.additionList.append(addList);
339  }
340  } else if (mapKey == QLatin1String("removals")) {
341  const QVariantList lst = mapIt.value().toList();
342  const QVector<Removal> removeList = d->parseRemovals(lst);
343  if (!removeList.isEmpty()) {
344  databaseInfo.removalList.append(removeList);
345  }
346  } else if (mapKey == QLatin1String("checksum")) {
347  QMapIterator<QString, QVariant> mapCheckSum(mapIt.value().toMap());
348  while (mapCheckSum.hasNext()) {
349  mapCheckSum.next();
350  if (mapCheckSum.key() == QLatin1String("sha256")) {
351  databaseInfo.sha256 = mapCheckSum.value().toByteArray();
352  } else {
353  qCDebug(WEBENGINEVIEWER_LOG) << "Invalid checksum key" << mapCheckSum.key();
354  }
355  }
356  } else if (mapKey == QLatin1String("newClientState")) {
357  databaseInfo.newClientState = mapIt.value().toString();
358  } else if (mapKey == QLatin1String("platformType")) {
359  databaseInfo.platformType = mapIt.value().toString();
360  } else if (mapKey == QLatin1String("responseType")) {
361  const QString str = mapIt.value().toString();
362  if (str == QLatin1String("FULL_UPDATE")) {
363  databaseInfo.responseType = UpdateDataBaseInfo::FullUpdate;
364  } else if (str == QLatin1String("PARTIAL_UPDATE")) {
365  databaseInfo.responseType = UpdateDataBaseInfo::PartialUpdate;
366  } else {
367  qCDebug(WEBENGINEVIEWER_LOG) << " unknown responsetype " << str;
368  databaseInfo.responseType = UpdateDataBaseInfo::Unknown;
369  }
370  } else if (mapKey == QLatin1String("threatEntryType")) {
371  databaseInfo.threatEntryType = mapIt.value().toString();
372  } else if (mapKey == QLatin1String("threatType")) {
373  databaseInfo.threatType = mapIt.value().toString();
374  } else {
375  qCDebug(WEBENGINEVIEWER_LOG) << " unknown key " << mapKey;
376  }
377  }
378  }
379  }
380  } else if (i.key() == QLatin1String("minimumWaitDuration")) {
381  databaseInfo.minimumWaitDuration = i.value().toString();
382  } else {
383  qCDebug(WEBENGINEVIEWER_LOG) << " map key unknown " << i.key();
384  }
385  }
386  Q_EMIT finished(databaseInfo, ValidData);
387  }
388  }
389  deleteLater();
390 }
391 
392 void CreatePhishingUrlDataBaseJob::setContraintsCompressionType(CreatePhishingUrlDataBaseJob::ContraintsCompressionType type)
393 {
394  d->mContraintsCompressionType = type;
395 }
QJsonDocument fromJson(const QByteArray &json, QJsonParseError *error)
bool canConvert(int targetTypeId) const const
QMap::const_iterator cend() const const
void sslErrors(QNetworkReply *reply, const QList< QSslError > &errors)
QByteArray toJson() const const
void append(const T &value)
std::optional< QSqlQuery > query(const QString &queryStatement)
QString errorString() const const
QObject * sender() const const
void reserve(int alloc)
void ignoreSslErrors(const QList< QSslError > &errors)
QJsonDocument fromVariant(const QVariant &variant)
bool isNull() const const
void addQueryItem(const QString &key, const QString &value)
int count(const T &value) const const
void append(const T &value)
PartitionTable::TableType type
QMapIterator::Item next()
QByteArray readAll()
void deleteLater()
const T & value() const const
QVariant toVariant() const const
void errorOccurred(QNetworkReply::NetworkError code)
QMap::const_iterator cbegin() const const
QMap< QString, QVariant > toMap() const const
QByteArray fromBase64(const QByteArray &base64, QByteArray::Base64Options options)
bool isEmpty() const const
void finished(QNetworkReply *reply)
void setQuery(const QString &query, QUrl::ParsingMode mode)
QMetaObject::Connection connect(const QObject *sender, const char *signal, const QObject *receiver, const char *method, Qt::ConnectionType type)
T qobject_cast(QObject *object)
QFuture< void > map(Sequence &sequence, MapFunctor function)
Q_EMITQ_EMIT
const T value(const Key &key, const T &defaultValue) const const
This file is part of the KDE documentation.
Documentation copyright © 1996-2021 The KDE developers.
Generated on Sun Dec 5 2021 23:04:53 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.