Messagelib

createphishingurldatabasejob.cpp
1 /*
2  SPDX-FileCopyrightText: 2016-2023 Laurent Montel <[email protected]>
3 
4  SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6 
7 #include "createphishingurldatabasejob.h"
8 #include "checkphishingurlutil.h"
9 #include "updatedatabaseinfo.h"
10 #include "webengineviewer_debug.h"
11 #include <PimCommon/NetworkManager>
12 
13 #include <QJsonDocument>
14 #include <QUrlQuery>
15 
16 using namespace WebEngineViewer;
17 
18 WEBENGINEVIEWER_EXPORT bool webengineview_useCompactJson_CreatePhishingUrlDataBaseJob = true;
19 
20 class WebEngineViewer::CreatePhishingUrlDataBaseJobPrivate
21 {
22 public:
23  Q_REQUIRED_RESULT UpdateDataBaseInfo::CompressionType parseCompressionType(const QString &str);
24  Q_REQUIRED_RESULT RiceDeltaEncoding parseRiceDeltaEncoding(const QMap<QString, QVariant> &map);
25  Q_REQUIRED_RESULT QVector<Removal> parseRemovals(const QVariantList &lst);
26  Q_REQUIRED_RESULT QVector<Addition> parseAdditions(const QVariantList &lst);
27  QString mDataBaseState;
28  CreatePhishingUrlDataBaseJob::ContraintsCompressionType mContraintsCompressionType = CreatePhishingUrlDataBaseJob::RawCompression;
29  CreatePhishingUrlDataBaseJob::DataBaseDownloadType mDataBaseDownloadNeeded = CreatePhishingUrlDataBaseJob::FullDataBase;
30  QNetworkAccessManager *mNetworkAccessManager = nullptr;
31 };
32 
33 CreatePhishingUrlDataBaseJob::CreatePhishingUrlDataBaseJob(QObject *parent)
34  : QObject(parent)
35  , d(new CreatePhishingUrlDataBaseJobPrivate)
36 {
37  d->mNetworkAccessManager = new QNetworkAccessManager(this);
38  d->mNetworkAccessManager->setRedirectPolicy(QNetworkRequest::NoLessSafeRedirectPolicy);
39  d->mNetworkAccessManager->setStrictTransportSecurityEnabled(true);
40  d->mNetworkAccessManager->enableStrictTransportSecurityStore(true);
41 
42  connect(d->mNetworkAccessManager, &QNetworkAccessManager::finished, this, &CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished);
43  connect(d->mNetworkAccessManager, &QNetworkAccessManager::sslErrors, this, &CreatePhishingUrlDataBaseJob::slotSslErrors);
44 }
45 
46 CreatePhishingUrlDataBaseJob::~CreatePhishingUrlDataBaseJob() = default;
47 
48 void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)
49 {
50  qCDebug(WEBENGINEVIEWER_LOG) << " void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)" << error.count();
51  reply->ignoreSslErrors(error);
52 }
53 
54 void CreatePhishingUrlDataBaseJob::start()
55 {
56  if (!PimCommon::NetworkManager::self()->isOnline()) {
57  Q_EMIT finished(UpdateDataBaseInfo(), BrokenNetwork);
58  deleteLater();
59  } else {
60  QUrlQuery query;
61  query.addQueryItem(QStringLiteral("key"), WebEngineViewer::CheckPhishingUrlUtil::apiKey());
62  QUrl safeUrl = QUrl(QStringLiteral("https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch"));
63  safeUrl.setQuery(query);
64  // qCDebug(WEBENGINEVIEWER_LOG) << " safeUrl" << safeUrl;
65  QNetworkRequest request(safeUrl);
66  request.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json"));
67 
68  const QByteArray baPostData = jsonRequest();
69  Q_EMIT debugJson(baPostData);
70  qCDebug(WEBENGINEVIEWER_LOG) << " postData.toJson()" << baPostData;
71  // curl -H "Content-Type: application/json" -X POST -d
72  // '{"client":{"clientId":"KDE","clientVersion":"5.4.0"},"threatInfo":{"platformTypes":["WINDOWS"],"threatEntries":[{"url":"http://www.kde.org"}],"threatEntryTypes":["URL"],"threatTypes":["MALWARE"]}}'
73  // https://safebrowsing.googleapis.com/v4/threatMatches:find?key=AIzaSyBS62pXATjabbH2RM_jO2EzDg1mTMHlnyo
74  QNetworkReply *reply = d->mNetworkAccessManager->post(request, baPostData);
75  connect(reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::errorOccurred), this, &CreatePhishingUrlDataBaseJob::slotError);
76  }
77 }
78 
79 void CreatePhishingUrlDataBaseJob::setDataBaseState(const QString &value)
80 {
81  d->mDataBaseState = value;
82 }
83 
84 void CreatePhishingUrlDataBaseJob::slotError(QNetworkReply::NetworkError error)
85 {
86  auto reply = qobject_cast<QNetworkReply *>(sender());
87  qWarning() << " error " << error << " error string : " << reply->errorString();
88  reply->deleteLater();
89  deleteLater();
90 }
91 
92 QByteArray CreatePhishingUrlDataBaseJob::jsonRequest() const
93 {
94 #if 0
95  {
96  "client" : {
97  "clientId" : "yourcompanyname",
98  "clientVersion" : "1.5.2"
99  },
100  "listUpdateRequests" : [{
101  "threatType" : "MALWARE",
102  "platformType" : "WINDOWS",
103  "threatEntryType" : "URL",
104  "state" : "Gg4IBBADIgYQgBAiAQEoAQ==",
105  "constraints" : {
106  "maxUpdateEntries" : 2048,
107  "maxDatabaseEntries" : 4096,
108  "region" : "US",
109  "supportedCompressions" : ["RAW"]
110  }
111  }]
112  }
113 #endif
114  QVariantMap clientMap;
115  QVariantMap map;
116 
117  clientMap.insert(QStringLiteral("clientId"), QStringLiteral("KDE"));
118  clientMap.insert(QStringLiteral("clientVersion"), CheckPhishingUrlUtil::versionApps());
119  map.insert(QStringLiteral("client"), clientMap);
120 
121  QVariantList listUpdateRequests;
122 
123  QVariantMap threatMap;
124  threatMap.insert(QStringLiteral("platformType"), QStringLiteral("WINDOWS"));
125  threatMap.insert(QStringLiteral("threatType"), QStringLiteral("MALWARE"));
126  threatMap.insert(QStringLiteral("threatEntryType"), QStringLiteral("URL"));
127 
128  // Contrainsts
129  QVariantMap contraintsMap;
130  QVariantList contraintsCompressionList;
131  QString compressionStr;
132  switch (d->mContraintsCompressionType) {
133  case RiceCompression:
134  compressionStr = QStringLiteral("RICE");
135  break;
136  case RawCompression:
137  compressionStr = QStringLiteral("RAW");
138  break;
139  }
140  contraintsCompressionList.append(compressionStr);
141  contraintsMap.insert(QStringLiteral("supportedCompressions"), contraintsCompressionList);
142  threatMap.insert(QStringLiteral("constraints"), contraintsMap);
143 
144  // Define state when we want to define update database. Empty is full.
145  switch (d->mDataBaseDownloadNeeded) {
146  case FullDataBase:
147  qCDebug(WEBENGINEVIEWER_LOG) << " full update";
148  threatMap.insert(QStringLiteral("state"), QString());
149  break;
150  case UpdateDataBase:
151  qCDebug(WEBENGINEVIEWER_LOG) << " update database";
152  if (d->mDataBaseState.isEmpty()) {
153  qCWarning(WEBENGINEVIEWER_LOG) << "Partial Download asked but database set is empty";
154  }
155  threatMap.insert(QStringLiteral("state"), d->mDataBaseState);
156  break;
157  }
158 
159  listUpdateRequests.append(threatMap);
160 
161  map.insert(QStringLiteral("listUpdateRequests"), listUpdateRequests);
162 
163  const QJsonDocument postData = QJsonDocument::fromVariant(map);
164  const QByteArray baPostData = postData.toJson(webengineview_useCompactJson_CreatePhishingUrlDataBaseJob ? QJsonDocument::Compact : QJsonDocument::Indented);
165  return baPostData;
166 }
167 
168 void CreatePhishingUrlDataBaseJob::setDataBaseDownloadNeeded(CreatePhishingUrlDataBaseJob::DataBaseDownloadType type)
169 {
170  d->mDataBaseDownloadNeeded = type;
171 }
172 
173 void CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished(QNetworkReply *reply)
174 {
175  const QByteArray returnValue(reply->readAll());
176  Q_EMIT debugJsonResult(returnValue);
177  parseResult(returnValue);
178  reply->deleteLater();
179 }
180 
181 RiceDeltaEncoding CreatePhishingUrlDataBaseJobPrivate::parseRiceDeltaEncoding(const QMap<QString, QVariant> &map)
182 {
183  RiceDeltaEncoding riceDeltaEncodingTmp;
184  QMap<QString, QVariant>::const_iterator riceHashesIt = map.cbegin();
185  const QMap<QString, QVariant>::const_iterator riceHashesItEnd = map.cend();
186  for (; riceHashesIt != riceHashesItEnd; ++riceHashesIt) {
187  const QString key = riceHashesIt.key();
188  if (key == QLatin1String("firstValue")) {
189  riceDeltaEncodingTmp.firstValue = riceHashesIt.value().toByteArray();
190  } else if (key == QLatin1String("riceParameter")) {
191  riceDeltaEncodingTmp.riceParameter = riceHashesIt.value().toInt();
192  } else if (key == QLatin1String("numEntries")) {
193  riceDeltaEncodingTmp.numberEntries = riceHashesIt.value().toInt();
194  } else if (key == QLatin1String("encodedData")) {
195  riceDeltaEncodingTmp.encodingData = riceHashesIt.value().toByteArray();
196  } else {
197  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRiceDeltaEncoding unknown riceDeltaEncoding key " << key;
198  }
199  }
200  return riceDeltaEncodingTmp;
201 }
202 
203 QVector<Addition> CreatePhishingUrlDataBaseJobPrivate::parseAdditions(const QVariantList &lst)
204 {
205  QVector<Addition> additionList;
206  for (const QVariant &v : lst) {
207  if (v.canConvert<QVariantMap>()) {
208  QMapIterator<QString, QVariant> mapIt(v.toMap());
209  Addition tmp;
210  while (mapIt.hasNext()) {
211  mapIt.next();
212  const QString keyStr = mapIt.key();
213  if (keyStr == QLatin1String("compressionType")) {
214  tmp.compressionType = parseCompressionType(mapIt.value().toString());
215  } else if (keyStr == QLatin1String("riceHashes")) {
216  RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
217  if (riceDeltaEncodingTmp.isValid()) {
218  tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
219  }
220  } else if (keyStr == QLatin1String("rawHashes")) {
221  QMapIterator<QString, QVariant> rawHashesIt(mapIt.value().toMap());
222  while (rawHashesIt.hasNext()) {
223  rawHashesIt.next();
224  const QString key = rawHashesIt.key();
225  if (key == QLatin1String("rawHashes")) {
226  tmp.hashString = QByteArray::fromBase64(rawHashesIt.value().toByteArray());
227  } else if (key == QLatin1String("prefixSize")) {
228  tmp.prefixSize = rawHashesIt.value().toInt();
229  } else {
230  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown rawHashes key " << key;
231  }
232  }
233  } else {
234  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown mapIt.key() " << keyStr;
235  }
236  }
237  if (tmp.isValid()) {
238  additionList.append(tmp);
239  }
240  } else {
241  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions not parsing type " << v.typeName();
242  }
243  }
244  return additionList;
245 }
246 
247 UpdateDataBaseInfo::CompressionType CreatePhishingUrlDataBaseJobPrivate::parseCompressionType(const QString &str)
248 {
249  UpdateDataBaseInfo::CompressionType type(UpdateDataBaseInfo::UnknownCompression);
250  if (str == QLatin1String("COMPRESSION_TYPE_UNSPECIFIED")) {
251  type = UpdateDataBaseInfo::UnknownCompression;
252  } else if (str == QLatin1String("RICE")) {
253  type = UpdateDataBaseInfo::RiceCompression;
254  } else if (str == QLatin1String("RAW")) {
255  type = UpdateDataBaseInfo::RawCompression;
256  } else {
257  qCWarning(WEBENGINEVIEWER_LOG) << "CreatePhishingUrlDataBaseJob::parseCompressionType unknown compression type " << str;
258  }
259  return type;
260 }
261 
262 QVector<Removal> CreatePhishingUrlDataBaseJobPrivate::parseRemovals(const QVariantList &lst)
263 {
264  QVector<Removal> removalList;
265  for (const QVariant &v : lst) {
266  if (v.canConvert<QVariantMap>()) {
267  Removal tmp;
268  QMapIterator<QString, QVariant> mapIt(v.toMap());
269  while (mapIt.hasNext()) {
270  mapIt.next();
271  const QString keyStr = mapIt.key();
272  if (keyStr == QLatin1String("compressionType")) {
273  tmp.compressionType = parseCompressionType(mapIt.value().toString());
274  } else if (keyStr == QLatin1String("riceIndices")) {
275  RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
276  if (riceDeltaEncodingTmp.isValid()) {
277  tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
278  }
279  } else if (keyStr == QLatin1String("rawIndices")) {
280  const QVariantMap map = mapIt.value().toMap();
281  QMapIterator<QString, QVariant> rawIndicesIt(map);
282  while (rawIndicesIt.hasNext()) {
283  rawIndicesIt.next();
284  if (rawIndicesIt.key() == QLatin1String("indices")) {
285  const QVariantList rawList = rawIndicesIt.value().toList();
286  QList<quint32> indexList;
287  indexList.reserve(rawList.count());
288  for (const QVariant &var : rawList) {
289  indexList.append(var.toUInt());
290  }
291  tmp.indexes = indexList;
292  } else {
293  qCDebug(WEBENGINEVIEWER_LOG) << "rawIndicesIt.key() unknown " << rawIndicesIt.key();
294  }
295  }
296  } else {
297  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals unknown mapIt.key() " << keyStr;
298  }
299  }
300  if (tmp.isValid()) {
301  removalList.append(tmp);
302  }
303  } else {
304  qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals not parsing type " << v.typeName();
305  }
306  }
307  return removalList;
308 }
309 
310 void CreatePhishingUrlDataBaseJob::parseResult(const QByteArray &value)
311 {
312  UpdateDataBaseInfo databaseInfo;
313  QJsonDocument document = QJsonDocument::fromJson(value);
314  if (document.isNull()) {
315  Q_EMIT finished(databaseInfo, InvalidData);
316  } else {
317  const QVariantMap answer = document.toVariant().toMap();
318  if (answer.isEmpty()) {
319  Q_EMIT finished(databaseInfo, InvalidData);
320  } else {
321  QMapIterator<QString, QVariant> i(answer);
322  while (i.hasNext()) {
323  i.next();
324  if (i.key() == QLatin1String("listUpdateResponses")) {
325  const QVariantList info = i.value().toList();
326  if (info.count() == 1) {
327  const QVariant infoVar = info.at(0);
328  if (infoVar.canConvert<QVariantMap>()) {
329  QMapIterator<QString, QVariant> mapIt(infoVar.toMap());
330  while (mapIt.hasNext()) {
331  mapIt.next();
332  const QString mapKey = mapIt.key();
333  if (mapKey == QLatin1String("additions")) {
334  const QVariantList lst = mapIt.value().toList();
335  const QVector<Addition> addList = d->parseAdditions(lst);
336  if (!addList.isEmpty()) {
337  databaseInfo.additionList.append(addList);
338  }
339  } else if (mapKey == QLatin1String("removals")) {
340  const QVariantList lst = mapIt.value().toList();
341  const QVector<Removal> removeList = d->parseRemovals(lst);
342  if (!removeList.isEmpty()) {
343  databaseInfo.removalList.append(removeList);
344  }
345  } else if (mapKey == QLatin1String("checksum")) {
346  QMapIterator<QString, QVariant> mapCheckSum(mapIt.value().toMap());
347  while (mapCheckSum.hasNext()) {
348  mapCheckSum.next();
349  if (mapCheckSum.key() == QLatin1String("sha256")) {
350  databaseInfo.sha256 = mapCheckSum.value().toByteArray();
351  } else {
352  qCDebug(WEBENGINEVIEWER_LOG) << "Invalid checksum key" << mapCheckSum.key();
353  }
354  }
355  } else if (mapKey == QLatin1String("newClientState")) {
356  databaseInfo.newClientState = mapIt.value().toString();
357  } else if (mapKey == QLatin1String("platformType")) {
358  databaseInfo.platformType = mapIt.value().toString();
359  } else if (mapKey == QLatin1String("responseType")) {
360  const QString str = mapIt.value().toString();
361  if (str == QLatin1String("FULL_UPDATE")) {
362  databaseInfo.responseType = UpdateDataBaseInfo::FullUpdate;
363  } else if (str == QLatin1String("PARTIAL_UPDATE")) {
364  databaseInfo.responseType = UpdateDataBaseInfo::PartialUpdate;
365  } else {
366  qCDebug(WEBENGINEVIEWER_LOG) << " unknown responsetype " << str;
367  databaseInfo.responseType = UpdateDataBaseInfo::Unknown;
368  }
369  } else if (mapKey == QLatin1String("threatEntryType")) {
370  databaseInfo.threatEntryType = mapIt.value().toString();
371  } else if (mapKey == QLatin1String("threatType")) {
372  databaseInfo.threatType = mapIt.value().toString();
373  } else {
374  qCDebug(WEBENGINEVIEWER_LOG) << " unknown key " << mapKey;
375  }
376  }
377  }
378  }
379  } else if (i.key() == QLatin1String("minimumWaitDuration")) {
380  databaseInfo.minimumWaitDuration = i.value().toString();
381  } else {
382  qCDebug(WEBENGINEVIEWER_LOG) << " map key unknown " << i.key();
383  }
384  }
385  Q_EMIT finished(databaseInfo, ValidData);
386  }
387  }
388  deleteLater();
389 }
390 
391 void CreatePhishingUrlDataBaseJob::setContraintsCompressionType(CreatePhishingUrlDataBaseJob::ContraintsCompressionType type)
392 {
393  d->mContraintsCompressionType = type;
394 }
QList::append
void append(const T &value)
QIODevice::errorString
QString errorString() const const
QNetworkReply::errorOccurred
void errorOccurred(QNetworkReply::NetworkError code)
QVector::isEmpty
bool isEmpty() const const
Akonadi::Server::QueryCache::query
std::optional< QSqlQuery > query(const QString &queryStatement)
QJsonDocument::fromJson
QJsonDocument fromJson(const QByteArray &json, QJsonParseError *error)
QObject::Q_EMIT
Q_EMITQ_EMIT
QJsonDocument::isNull
bool isNull() const const
QJsonDocument::toVariant
QVariant toVariant() const const
QVector::append
void append(const T &value)
QObject::sender
QObject * sender() const const
QObject::connect
QMetaObject::Connection connect(const QObject *sender, const char *signal, const QObject *receiver, const char *method, Qt::ConnectionType type)
QList::reserve
void reserve(int alloc)
QNetworkAccessManager::finished
void finished(QNetworkReply *reply)
QObject::deleteLater
void deleteLater()
QNetworkAccessManager::sslErrors
void sslErrors(QNetworkReply *reply, const QList< QSslError > &errors)
QByteArray::fromBase64
QByteArray fromBase64(const QByteArray &base64, QByteArray::Base64Options options)
KMessageBox::error
void error(QWidget *parent, const QString &text, const QString &title, const KGuiItem &buttonOk, Options options=Notify)
QVariant::toMap
QMap< QString, QVariant > toMap() const const
QUrl::setQuery
void setQuery(const QString &query, QUrl::ParsingMode mode)
QVariant::canConvert
bool canConvert(int targetTypeId) const const
QNetworkReply::ignoreSslErrors
void ignoreSslErrors(const QList< QSslError > &errors)
QJsonDocument::toJson
QByteArray toJson() const const
QIODevice::readAll
QByteArray readAll()
QJsonDocument::fromVariant
QJsonDocument fromVariant(const QVariant &variant)
QtConcurrent::map
QFuture< void > map(Sequence &sequence, MapFunctor function)
This file is part of the KDE documentation.
Documentation copyright © 1996-2023 The KDE developers.
Generated on Thu Mar 30 2023 04:03:00 by doxygen 1.8.17 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.