Akonadi

itemretriever.cpp
1 /*
2  SPDX-FileCopyrightText: 2009 Volker Krause <[email protected]>
3  SPDX-FileCopyrightText: 2010 Milian Wolff <[email protected]>
4 
5  SPDX-License-Identifier: LGPL-2.0-or-later
6 */
7 
8 #include "itemretriever.h"
9 
10 #include "akonadi.h"
11 #include "connection.h"
12 #include "storage/datastore.h"
13 #include "storage/itemqueryhelper.h"
14 #include "storage/itemretrievalmanager.h"
15 #include "storage/itemretrievalrequest.h"
16 #include "storage/parthelper.h"
17 #include "storage/parttypehelper.h"
18 #include "storage/querybuilder.h"
19 #include "storage/selectquerybuilder.h"
20 #include "utils.h"
21 
22 #include <shared/akranges.h>
23 #include <private/protocol_p.h>
24 
25 #include <QEventLoop>
26 
27 #include "akonadiserver_debug.h"
28 
29 using namespace Akonadi;
30 using namespace Akonadi::Server;
31 using namespace AkRanges;
32 
33 Q_DECLARE_METATYPE(ItemRetrievalResult)
34 
35 ItemRetriever::ItemRetriever(ItemRetrievalManager &manager, Connection *connection, const CommandContext &context)
36  : mItemRetrievalManager(manager)
37  , mConnection(connection)
38  , mContext(context)
39  , mFullPayload(false)
40  , mRecursive(false)
41  , mCanceled(false)
42 {
43  qRegisterMetaType<ItemRetrievalResult>("Akonadi::Server::ItemRetrievalResult");
44  if (mConnection) {
45  connect(mConnection, &Connection::disconnected,
46  this, [this]() {
47  mCanceled = true;
48  });
49  }
50 }
51 
52 Connection *ItemRetriever::connection() const
53 {
54  return mConnection;
55 }
56 
57 void ItemRetriever::setRetrieveParts(const QVector<QByteArray> &parts)
58 {
59  mParts = parts;
60  std::sort(mParts.begin(), mParts.end());
61  mParts.erase(std::unique(mParts.begin(), mParts.end()), mParts.end());
62 
63  // HACK, we need a full payload available flag in PimItem
64  if (mFullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
65  mParts.append(AKONADI_PARAM_PLD_RFC822);
66  }
67 }
68 
69 void ItemRetriever::setItemSet(const ImapSet &set, const Collection &collection)
70 {
71  mItemSet = set;
72  mCollection = collection;
73 }
74 
75 void ItemRetriever::setItemSet(const ImapSet &set, bool isUid)
76 {
77  if (!isUid && mContext.collectionId() >= 0) {
78  setItemSet(set, mContext.collection());
79  } else {
80  setItemSet(set);
81  }
82 }
83 
84 void ItemRetriever::setItem(Entity::Id id)
85 {
86  ImapSet set;
87  set.add(ImapInterval(id, id));
88  mItemSet = set;
89  mCollection = Collection();
90 }
91 
92 void ItemRetriever::setRetrieveFullPayload(bool fullPayload)
93 {
94  mFullPayload = fullPayload;
95  // HACK, we need a full payload available flag in PimItem
96  if (fullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
97  mParts.append(AKONADI_PARAM_PLD_RFC822);
98  }
99 }
100 
101 void ItemRetriever::setCollection(const Collection &collection, bool recursive)
102 {
103  mCollection = collection;
104  mItemSet = ImapSet();
105  mRecursive = recursive;
106 }
107 
108 void ItemRetriever::setScope(const Scope &scope)
109 {
110  mScope = scope;
111 }
112 
113 Scope ItemRetriever::scope() const
114 {
115  return mScope;
116 }
117 
118 void ItemRetriever::setChangedSince(const QDateTime &changedSince)
119 {
120  mChangedSince = changedSince;
121 }
122 
123 QVector<QByteArray> ItemRetriever::retrieveParts() const
124 {
125  return mParts;
126 }
127 
128 enum QueryColumns {
129  PimItemIdColumn,
130 
131  CollectionIdColumn,
132  ResourceIdColumn,
133 
134  PartTypeNameColumn,
135  PartDatasizeColumn
136 };
137 
138 QSqlQuery ItemRetriever::buildQuery() const
139 {
140  QueryBuilder qb(PimItem::tableName());
141 
142  qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
143 
144  qb.addJoin(QueryBuilder::LeftJoin, Part::tableName(), PimItem::idFullColumnName(), Part::pimItemIdFullColumnName());
145 
146  Query::Condition partTypeJoinCondition;
147  partTypeJoinCondition.addColumnCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartType::idFullColumnName());
148  if (!mFullPayload && !mParts.isEmpty()) {
149  partTypeJoinCondition.addCondition(PartTypeHelper::conditionFromFqNames(mParts));
150  }
151  partTypeJoinCondition.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
152  qb.addJoin(QueryBuilder::LeftJoin, PartType::tableName(), partTypeJoinCondition);
153 
154  qb.addColumn(PimItem::idFullColumnName());
155  qb.addColumn(PimItem::collectionIdFullColumnName());
156  qb.addColumn(Collection::resourceIdFullColumnName());
157  qb.addColumn(PartType::nameFullColumnName());
158  qb.addColumn(Part::datasizeFullColumnName());
159 
160  if (!mItemSet.isEmpty() || mCollection.isValid()) {
161  ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
162  } else {
163  ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
164  }
165 
166  // prevent a resource to trigger item retrieval from itself
167  if (mConnection) {
168  const Resource res = Resource::retrieveByName(QString::fromUtf8(mConnection->sessionId()));
169  if (res.isValid()) {
170  qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::NotEquals, res.id());
171  }
172  }
173 
174  if (mChangedSince.isValid()) {
175  qb.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual,
176  mChangedSince.toUTC());
177  }
178 
179  qb.addSortColumn(PimItem::idFullColumnName(), Query::Ascending);
180 
181  if (!qb.exec()) {
182  mLastError = "Unable to retrieve items";
183  throw ItemRetrieverException(mLastError);
184  }
185 
186  qb.query().next();
187 
188  return qb.query();
189 }
190 
191 namespace
192 {
193 bool hasAllParts(const ItemRetrievalRequest &req, const QSet<QByteArray> &availableParts)
194 {
195  return std::all_of(req.parts.begin(), req.parts.end(), [&availableParts](const auto &part) {
196  return availableParts.contains(part);
197  });
198 }
199 }
200 
201 bool ItemRetriever::runItemRetrievalRequests(std::list<ItemRetrievalRequest> requests) // clazy:exclude=function-args-by-ref
202 {
203  QEventLoop eventLoop;
204  std::vector<ItemRetrievalRequest::Id> pendingRequests;
205  connect(&mItemRetrievalManager, &ItemRetrievalManager::requestFinished,
206  this, [this, &eventLoop, &pendingRequests](const ItemRetrievalResult &result) { // clazy:exclude=lambda-in-connect
207  const auto requestId = std::find(pendingRequests.begin(), pendingRequests.end(), result.request.id);
208  if (requestId != pendingRequests.end()) {
209  if (mCanceled) {
210  eventLoop.exit(1);
211  } else if (result.errorMsg.has_value()) {
212  mLastError = result.errorMsg->toUtf8();
213  eventLoop.exit(1);
214  } else {
215  Q_EMIT itemsRetrieved(result.request.ids);
216  pendingRequests.erase(requestId);
217  if (pendingRequests.empty()) {
218  eventLoop.quit();
219  }
220  }
221  }
222  });
223 
224  if (mConnection) {
225  connect(mConnection, &Connection::connectionClosing,
226  &eventLoop, [&eventLoop]() { eventLoop.exit(1); });
227  }
228 
229  for (auto &&request : requests) {
230  if ((!mFullPayload && request.parts.isEmpty()) || request.ids.isEmpty()) {
231  continue;
232  }
233 
234  // TODO: how should we handle retrieval errors here? so far they have been ignored,
235  // which makes sense in some cases, do we need a command parameter for this?
236  try {
237  // Request is deleted inside ItemRetrievalManager, so we need to take
238  // a copy here
239  //const auto ids = request->ids;
240  pendingRequests.push_back(request.id);
241  mItemRetrievalManager.requestItemDelivery(std::move(request));
242  } catch (const ItemRetrieverException &e) {
243  qCCritical(AKONADISERVER_LOG) << e.type() << ": " << e.what();
244  mLastError = e.what();
245  return false;
246  }
247  }
248 
249  if (!pendingRequests.empty()) {
250  if (eventLoop.exec()) {
251  return false;
252  }
253  }
254 
255  return true;
256 }
257 
258 std::optional<ItemRetriever::PreparedRequests> ItemRetriever::prepareRequests(QSqlQuery &query, const QByteArrayList &parts)
259 {
260  QHash<qint64, QString> resourceIdNameCache;
261  std::list<ItemRetrievalRequest> requests;
262  QHash<qint64 /* collection */, decltype(requests)::iterator> colRequests;
263  QHash<qint64 /* item */, decltype(requests)::iterator> itemRequests;
264  QVector<qint64> readyItems;
265  qint64 prevPimItemId = -1;
266  QSet<QByteArray> availableParts;
267  decltype(requests)::iterator lastRequest = requests.end();
268  while (query.isValid()) {
269  const qint64 pimItemId = query.value(PimItemIdColumn).toLongLong();
270  const qint64 collectionId = query.value(CollectionIdColumn).toLongLong();
271  const qint64 resourceId = query.value(ResourceIdColumn).toLongLong();
272  const auto itemIter = itemRequests.constFind(pimItemId);
273 
274  if (Q_UNLIKELY(mCanceled)) {
275  return std::nullopt;
276  }
277 
278  if (pimItemId == prevPimItemId) {
279  if (query.value(PartTypeNameColumn).isNull()) {
280  // This is not the first part of the Item we saw, but LEFT JOIN PartTable
281  // returned a null row - that means the row is an ATR part
282  // which we don't care about
283  query.next();
284  continue;
285  }
286  } else {
287  if (lastRequest != requests.end()) {
288  if (hasAllParts(*lastRequest, availableParts)) {
289  // We went through all parts of a single item, if we have all
290  // parts available in the DB and they are not expired, then
291  // exclude this item from the retrieval
292  lastRequest->ids.removeOne(prevPimItemId);
293  itemRequests.remove(prevPimItemId);
294  readyItems.push_back(prevPimItemId);
295  }
296  }
297  availableParts.clear();
298  prevPimItemId = pimItemId;
299  }
300 
301  if (itemIter != itemRequests.constEnd()) {
302  lastRequest = itemIter.value();
303  } else {
304  const auto colIt = colRequests.find(collectionId);
305  lastRequest = (colIt == colRequests.end()) ? requests.end() : colIt.value();
306  if (lastRequest == requests.end() || lastRequest->ids.size() > 100) {
307  requests.emplace_front(ItemRetrievalRequest{});
308  lastRequest = requests.begin();
309  lastRequest->ids.push_back(pimItemId);
310  auto resIter = resourceIdNameCache.find(resourceId);
311  if (resIter == resourceIdNameCache.end()) {
312  resIter = resourceIdNameCache.insert(resourceId, Resource::retrieveById(resourceId).name());
313  }
314  lastRequest->resourceId = *resIter;
315  lastRequest->parts = parts;
316  colRequests.insert(collectionId, lastRequest);
317  itemRequests.insert(pimItemId, lastRequest);
318  } else {
319  lastRequest->ids.push_back(pimItemId);
320  itemRequests.insert(pimItemId, lastRequest);
321  colRequests.insert(collectionId, lastRequest);
322  }
323  }
324  Q_ASSERT(lastRequest != requests.end());
325 
326  if (query.value(PartTypeNameColumn).isNull()) {
327  // LEFT JOIN did not find anything, retrieve all parts
328  query.next();
329  continue;
330  }
331 
332  qint64 datasize = query.value(PartDatasizeColumn).toLongLong();
333  const QByteArray partName = Utils::variantToByteArray(query.value(PartTypeNameColumn));
334  Q_ASSERT(!partName.startsWith(AKONADI_PARAM_PLD));
335  if (datasize <= 0) {
336  // request update for this part
337  if (mFullPayload && !lastRequest->parts.contains(partName)) {
338  lastRequest->parts.push_back(partName);
339  }
340  } else {
341  // add the part to list of available parts, we will compare it with
342  // the list of request parts once we handle all parts of this item
343  availableParts.insert(partName);
344  }
345  query.next();
346  }
347  query.finish();
348 
349  // Post-check in case we only queried one item thus did not reach the check
350  // at the beginning of the while() loop above
351  if (lastRequest != requests.end() && hasAllParts(*lastRequest, availableParts)) {
352  lastRequest->ids.removeOne(prevPimItemId);
353  readyItems.push_back(prevPimItemId);
354  // No need to update the hashtable at this point
355  }
356 
357  return PreparedRequests{std::move(requests), std::move(readyItems)};
358 }
359 
360 bool ItemRetriever::exec()
361 {
362  if (mParts.isEmpty() && !mFullPayload) {
363  return true;
364  }
365 
366  verifyCache();
367 
368  QSqlQuery query = buildQuery();
369  const auto parts = mParts | Views::filter([](const auto &part) { return part.startsWith(AKONADI_PARAM_PLD); })
370  | Views::transform([](const auto &part) { return part.mid(4); })
371  | Actions::toQList;
372 
373  auto requests = prepareRequests(query, parts);
374  if (!requests.has_value()) {
375  return false;
376  }
377 
378  if (!requests->readyItems.isEmpty()) {
379  Q_EMIT itemsRetrieved(requests->readyItems);
380  }
381 
382  if (!runItemRetrievalRequests(std::move(requests->requests))) {
383  return false;
384  }
385 
386  // retrieve items in child collections if requested
387  bool result = true;
388  if (mRecursive && mCollection.isValid()) {
389  Q_FOREACH (const Collection &col, mCollection.children()) {
390  ItemRetriever retriever(mItemRetrievalManager, mConnection, mContext);
391  retriever.setCollection(col, mRecursive);
392  retriever.setRetrieveParts(mParts);
393  retriever.setRetrieveFullPayload(mFullPayload);
394  connect(&retriever, &ItemRetriever::itemsRetrieved,
395  this, &ItemRetriever::itemsRetrieved);
396  result = retriever.exec();
397  if (!result) {
398  break;
399  }
400  }
401  }
402 
403  return result;
404 }
405 
406 void ItemRetriever::verifyCache()
407 {
408  if (!connection() || !connection()->verifyCacheOnRetrieval()) {
409  return;
410  }
411 
413  qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
414  qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
415  qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant());
416  if (mScope.scope() != Scope::Invalid) {
417  ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
418  } else {
419  ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
420  }
421 
422  if (!qb.exec()) {
423  mLastError = QByteArrayLiteral("Unable to query parts.");
424  throw ItemRetrieverException(mLastError);
425  }
426 
427  const Part::List externalParts = qb.result();
428  for (Part part : externalParts) {
429  PartHelper::verify(part);
430  }
431 }
432 
433 QByteArray ItemRetriever::lastError() const
434 {
435  return mLastError;
436 }
qlonglong toLongLong(bool *ok) const const
bool verify(Part &part)
Verifies and if necessary fixes the external reference of this part.
Definition: parthelper.cpp:165
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
void addCondition(const Condition &condition)
Add a WHERE condition.
Definition: query.cpp:54
void setCollection(const Collection &collection, bool recursive=true)
Retrieve all items in the given collection.
void quit()
QHash::iterator insert(const Key &key, const T &value)
QSqlQuery & query()
Returns the query, only valid after exec().
Helper class for retrieving missing items parts from remote resources.
Definition: itemretriever.h:40
void itemSetToQuery(const ImapSet &set, QueryBuilder &qb, const Collection &collection=Collection())
Add conditions to qb for the given item set set.
Represents a collection of PIM items.
Definition: collection.h:63
QVector::iterator erase(QVector::iterator begin, QVector::iterator end)
bool startsWith(const QByteArray &ba) const const
NOTE: only supported for UPDATE and SELECT queries.
Definition: querybuilder.h:49
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
QSet::iterator insert(const T &value)
void addColumn(const QString &col)
Adds the given column to a select query.
QString fromUtf8(const char *str, int size)
bool isNull() const const
int exec(QEventLoop::ProcessEventsFlags flags)
QVariant value(int index) const const
Manages and processes item retrieval requests.
void addSortColumn(const QString &column, Query::SortOrder order=Query::Ascending)
Add sort column.
void exit(int returnCode)
bool next()
QVector< T > result()
Returns the result of this SELECT query.
void addColumnCondition(const QString &column, CompareOperator op, const QString &column2)
Add a WHERE condition which compares a column with another column.
Definition: query.cpp:22
void setScope(const Scope &scope)
Retrieve all items matching the given item scope.
NOTE: only supported for SELECT queries.
Definition: querybuilder.h:51
void push_back(char ch)
QList::iterator end()
QHash::iterator find(const Key &key)
bool contains(const T &value) const const
bool isValid() const const
Details of a single item retrieval request.
Helper class for creating and executing database SELECT queries.
void finish()
void insert(int i, const T &value)
Helper integration between Akonadi and Qt.
void push_back(const T &value)
Represents a WHERE condition tree.
Definition: query.h:64
QHash::iterator end()
void clear()
void scopeToQuery(const Scope &scope, const CommandContext &context, QueryBuilder &qb)
Add conditions to qb for the given item operation scope scope.
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:32
QList::iterator begin()
Query::Condition conditionFromFqNames(const QStringList &fqNames)
Returns a query condition that matches the given part type list.
bool exec()
Executes the query, returns true on success.
An Connection represents one connection of a client to the server.
Definition: connection.h:40
void addValueCondition(const QString &column, CompareOperator op, const QVariant &value)
Add a WHERE condition which compares a column with a given value.
Definition: query.cpp:12
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Tue Jul 7 2020 23:14:31 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.