Akonadi

itemretriever.cpp
1 /*
2  SPDX-FileCopyrightText: 2009 Volker Krause <[email protected]>
3  SPDX-FileCopyrightText: 2010 Milian Wolff <[email protected]>
4 
5  SPDX-License-Identifier: LGPL-2.0-or-later
6 */
7 
8 #include "itemretriever.h"
9 
10 #include "akonadi.h"
11 #include "connection.h"
12 #include "storage/datastore.h"
13 #include "storage/itemqueryhelper.h"
14 #include "storage/itemretrievalmanager.h"
15 #include "storage/itemretrievalrequest.h"
16 #include "storage/parthelper.h"
17 #include "storage/parttypehelper.h"
18 #include "storage/querybuilder.h"
19 #include "storage/selectquerybuilder.h"
20 #include "utils.h"
21 
22 #include <private/protocol_p.h>
23 #include <shared/akranges.h>
24 
25 #include <QEventLoop>
26 
27 #include "akonadiserver_debug.h"
28 
29 using namespace Akonadi;
30 using namespace Akonadi::Server;
31 using namespace AkRanges;
32 
33 Q_DECLARE_METATYPE(ItemRetrievalResult)
34 
35 ItemRetriever::ItemRetriever(ItemRetrievalManager &manager, Connection *connection, const CommandContext &context)
36  : mItemRetrievalManager(manager)
37  , mConnection(connection)
38  , mContext(context)
39  , mFullPayload(false)
40  , mRecursive(false)
41  , mCanceled(false)
42 {
43  qRegisterMetaType<ItemRetrievalResult>("Akonadi::Server::ItemRetrievalResult");
44  if (mConnection) {
45  connect(mConnection, &Connection::disconnected, this, [this]() {
46  mCanceled = true;
47  });
48  }
49 }
50 
51 Connection *ItemRetriever::connection() const
52 {
53  return mConnection;
54 }
55 
56 void ItemRetriever::setRetrieveParts(const QVector<QByteArray> &parts)
57 {
58  mParts = parts;
59  std::sort(mParts.begin(), mParts.end());
60  mParts.erase(std::unique(mParts.begin(), mParts.end()), mParts.end());
61 
62  // HACK, we need a full payload available flag in PimItem
63  if (mFullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
64  mParts.append(AKONADI_PARAM_PLD_RFC822);
65  }
66 }
67 
68 void ItemRetriever::setItemSet(const ImapSet &set, const Collection &collection)
69 {
70  mItemSet = set;
71  mCollection = collection;
72 }
73 
74 void ItemRetriever::setItemSet(const ImapSet &set, bool isUid)
75 {
76  if (!isUid && mContext.collectionId() >= 0) {
77  setItemSet(set, mContext.collection());
78  } else {
79  setItemSet(set);
80  }
81 }
82 
83 void ItemRetriever::setItem(Entity::Id id)
84 {
85  ImapSet set;
86  set.add(ImapInterval(id, id));
87  mItemSet = set;
88  mCollection = Collection();
89 }
90 
91 void ItemRetriever::setRetrieveFullPayload(bool fullPayload)
92 {
93  mFullPayload = fullPayload;
94  // HACK, we need a full payload available flag in PimItem
95  if (fullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
96  mParts.append(AKONADI_PARAM_PLD_RFC822);
97  }
98 }
99 
100 void ItemRetriever::setCollection(const Collection &collection, bool recursive)
101 {
102  mCollection = collection;
103  mItemSet = ImapSet();
104  mRecursive = recursive;
105 }
106 
107 void ItemRetriever::setScope(const Scope &scope)
108 {
109  mScope = scope;
110 }
111 
112 Scope ItemRetriever::scope() const
113 {
114  return mScope;
115 }
116 
117 void ItemRetriever::setChangedSince(const QDateTime &changedSince)
118 {
119  mChangedSince = changedSince;
120 }
121 
122 QVector<QByteArray> ItemRetriever::retrieveParts() const
123 {
124  return mParts;
125 }
126 
127 enum QueryColumns {
128  PimItemIdColumn,
129 
130  CollectionIdColumn,
131  ResourceIdColumn,
132 
133  PartTypeNameColumn,
134  PartDatasizeColumn
135 };
136 
137 QSqlQuery ItemRetriever::buildQuery() const
138 {
139  QueryBuilder qb(PimItem::tableName());
140 
141  qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
142 
143  qb.addJoin(QueryBuilder::LeftJoin, Part::tableName(), PimItem::idFullColumnName(), Part::pimItemIdFullColumnName());
144 
145  Query::Condition partTypeJoinCondition;
146  partTypeJoinCondition.addColumnCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartType::idFullColumnName());
147  if (!mFullPayload && !mParts.isEmpty()) {
148  partTypeJoinCondition.addCondition(PartTypeHelper::conditionFromFqNames(mParts));
149  }
150  partTypeJoinCondition.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
151  qb.addJoin(QueryBuilder::LeftJoin, PartType::tableName(), partTypeJoinCondition);
152 
153  qb.addColumn(PimItem::idFullColumnName());
154  qb.addColumn(PimItem::collectionIdFullColumnName());
155  qb.addColumn(Collection::resourceIdFullColumnName());
156  qb.addColumn(PartType::nameFullColumnName());
157  qb.addColumn(Part::datasizeFullColumnName());
158 
159  if (!mItemSet.isEmpty() || mCollection.isValid()) {
160  ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
161  } else {
162  ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
163  }
164 
165  // prevent a resource to trigger item retrieval from itself
166  if (mConnection) {
167  const Resource res = Resource::retrieveByName(QString::fromUtf8(mConnection->sessionId()));
168  if (res.isValid()) {
169  qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::NotEquals, res.id());
170  }
171  }
172 
173  if (mChangedSince.isValid()) {
174  qb.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual, mChangedSince.toUTC());
175  }
176 
177  qb.addSortColumn(PimItem::idFullColumnName(), Query::Ascending);
178 
179  if (!qb.exec()) {
180  mLastError = "Unable to retrieve items";
181  throw ItemRetrieverException(mLastError);
182  }
183 
184  qb.query().next();
185 
186  return qb.query();
187 }
188 
189 namespace
190 {
191 bool hasAllParts(const ItemRetrievalRequest &req, const QSet<QByteArray> &availableParts)
192 {
193  return std::all_of(req.parts.begin(), req.parts.end(), [&availableParts](const auto &part) {
194  return availableParts.contains(part);
195  });
196 }
197 }
198 
199 bool ItemRetriever::runItemRetrievalRequests(std::list<ItemRetrievalRequest> requests) // clazy:exclude=function-args-by-ref
200 {
201  QEventLoop eventLoop;
202  std::vector<ItemRetrievalRequest::Id> pendingRequests;
203  connect(&mItemRetrievalManager,
204  &ItemRetrievalManager::requestFinished,
205  this,
206  [this, &eventLoop, &pendingRequests](const ItemRetrievalResult &result) { // clazy:exclude=lambda-in-connect
207  const auto requestId = std::find(pendingRequests.begin(), pendingRequests.end(), result.request.id);
208  if (requestId != pendingRequests.end()) {
209  if (mCanceled) {
210  eventLoop.exit(1);
211  } else if (result.errorMsg.has_value()) {
212  mLastError = result.errorMsg->toUtf8();
213  eventLoop.exit(1);
214  } else {
215  Q_EMIT itemsRetrieved(result.request.ids);
216  pendingRequests.erase(requestId);
217  if (pendingRequests.empty()) {
218  eventLoop.quit();
219  }
220  }
221  }
222  });
223 
224  if (mConnection) {
225  connect(mConnection, &Connection::connectionClosing, &eventLoop, [&eventLoop]() {
226  eventLoop.exit(1);
227  });
228  }
229 
230  for (auto &&request : requests) {
231  if ((!mFullPayload && request.parts.isEmpty()) || request.ids.isEmpty()) {
232  continue;
233  }
234 
235  // TODO: how should we handle retrieval errors here? so far they have been ignored,
236  // which makes sense in some cases, do we need a command parameter for this?
237  try {
238  // Request is deleted inside ItemRetrievalManager, so we need to take
239  // a copy here
240  // const auto ids = request->ids;
241  pendingRequests.push_back(request.id);
242  mItemRetrievalManager.requestItemDelivery(std::move(request));
243  } catch (const ItemRetrieverException &e) {
244  qCCritical(AKONADISERVER_LOG) << e.type() << ": " << e.what();
245  mLastError = e.what();
246  return false;
247  }
248  }
249 
250  if (!pendingRequests.empty()) {
251  if (eventLoop.exec()) {
252  return false;
253  }
254  }
255 
256  return true;
257 }
258 
259 std::optional<ItemRetriever::PreparedRequests> ItemRetriever::prepareRequests(QSqlQuery &query, const QByteArrayList &parts)
260 {
261  QHash<qint64, QString> resourceIdNameCache;
262  std::list<ItemRetrievalRequest> requests;
263  QHash<qint64 /* collection */, decltype(requests)::iterator> colRequests;
264  QHash<qint64 /* item */, decltype(requests)::iterator> itemRequests;
265  QVector<qint64> readyItems;
266  qint64 prevPimItemId = -1;
267  QSet<QByteArray> availableParts;
268  auto lastRequest = requests.end();
269  while (query.isValid()) {
270  const qint64 pimItemId = query.value(PimItemIdColumn).toLongLong();
271  const qint64 collectionId = query.value(CollectionIdColumn).toLongLong();
272  const qint64 resourceId = query.value(ResourceIdColumn).toLongLong();
273  const auto itemIter = itemRequests.constFind(pimItemId);
274 
275  if (Q_UNLIKELY(mCanceled)) {
276  return std::nullopt;
277  }
278 
279  if (pimItemId == prevPimItemId) {
280  if (query.value(PartTypeNameColumn).isNull()) {
281  // This is not the first part of the Item we saw, but LEFT JOIN PartTable
282  // returned a null row - that means the row is an ATR part
283  // which we don't care about
284  query.next();
285  continue;
286  }
287  } else {
288  if (lastRequest != requests.end()) {
289  if (hasAllParts(*lastRequest, availableParts)) {
290  // We went through all parts of a single item, if we have all
291  // parts available in the DB and they are not expired, then
292  // exclude this item from the retrieval
293  lastRequest->ids.removeOne(prevPimItemId);
294  itemRequests.remove(prevPimItemId);
295  readyItems.push_back(prevPimItemId);
296  }
297  }
298  availableParts.clear();
299  prevPimItemId = pimItemId;
300  }
301 
302  if (itemIter != itemRequests.constEnd()) {
303  lastRequest = itemIter.value();
304  } else {
305  const auto colIt = colRequests.find(collectionId);
306  lastRequest = (colIt == colRequests.end()) ? requests.end() : colIt.value();
307  if (lastRequest == requests.end() || lastRequest->ids.size() > 100) {
308  requests.emplace_front(ItemRetrievalRequest{});
309  lastRequest = requests.begin();
310  lastRequest->ids.push_back(pimItemId);
311  auto resIter = resourceIdNameCache.find(resourceId);
312  if (resIter == resourceIdNameCache.end()) {
313  resIter = resourceIdNameCache.insert(resourceId, Resource::retrieveById(resourceId).name());
314  }
315  lastRequest->resourceId = *resIter;
316  lastRequest->parts = parts;
317  colRequests.insert(collectionId, lastRequest);
318  itemRequests.insert(pimItemId, lastRequest);
319  } else {
320  lastRequest->ids.push_back(pimItemId);
321  itemRequests.insert(pimItemId, lastRequest);
322  colRequests.insert(collectionId, lastRequest);
323  }
324  }
325  Q_ASSERT(lastRequest != requests.end());
326 
327  if (query.value(PartTypeNameColumn).isNull()) {
328  // LEFT JOIN did not find anything, retrieve all parts
329  query.next();
330  continue;
331  }
332 
333  qint64 datasize = query.value(PartDatasizeColumn).toLongLong();
334  const QByteArray partName = Utils::variantToByteArray(query.value(PartTypeNameColumn));
335  Q_ASSERT(!partName.startsWith(AKONADI_PARAM_PLD));
336  if (datasize <= 0) {
337  // request update for this part
338  if (mFullPayload && !lastRequest->parts.contains(partName)) {
339  lastRequest->parts.push_back(partName);
340  }
341  } else {
342  // add the part to list of available parts, we will compare it with
343  // the list of request parts once we handle all parts of this item
344  availableParts.insert(partName);
345  }
346  query.next();
347  }
348  query.finish();
349 
350  // Post-check in case we only queried one item thus did not reach the check
351  // at the beginning of the while() loop above
352  if (lastRequest != requests.end() && hasAllParts(*lastRequest, availableParts)) {
353  lastRequest->ids.removeOne(prevPimItemId);
354  readyItems.push_back(prevPimItemId);
355  // No need to update the hashtable at this point
356  }
357 
358  return PreparedRequests{std::move(requests), std::move(readyItems)};
359 }
360 
361 bool ItemRetriever::exec()
362 {
363  if (mParts.isEmpty() && !mFullPayload) {
364  return true;
365  }
366 
367  verifyCache();
368 
369  QSqlQuery query = buildQuery();
370  const auto parts = mParts | Views::filter([](const auto &part) {
371  return part.startsWith(AKONADI_PARAM_PLD);
372  })
373  | Views::transform([](const auto &part) {
374  return part.mid(4);
375  })
376  | Actions::toQList;
377 
378  auto requests = prepareRequests(query, parts);
379  if (!requests.has_value()) {
380  return false;
381  }
382 
383  if (!requests->readyItems.isEmpty()) {
384  Q_EMIT itemsRetrieved(requests->readyItems);
385  }
386 
387  if (!runItemRetrievalRequests(std::move(requests->requests))) {
388  return false;
389  }
390 
391  // retrieve items in child collections if requested
392  bool result = true;
393  if (mRecursive && mCollection.isValid()) {
394  Q_FOREACH (const Collection &col, mCollection.children()) {
395  ItemRetriever retriever(mItemRetrievalManager, mConnection, mContext);
396  retriever.setCollection(col, mRecursive);
397  retriever.setRetrieveParts(mParts);
398  retriever.setRetrieveFullPayload(mFullPayload);
399  connect(&retriever, &ItemRetriever::itemsRetrieved, this, &ItemRetriever::itemsRetrieved);
400  result = retriever.exec();
401  if (!result) {
402  break;
403  }
404  }
405  }
406 
407  return result;
408 }
409 
410 void ItemRetriever::verifyCache()
411 {
412  if (!connection() || !connection()->verifyCacheOnRetrieval()) {
413  return;
414  }
415 
417  qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
418  qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
419  qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant());
420  if (mScope.scope() != Scope::Invalid) {
421  ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
422  } else {
423  ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
424  }
425 
426  if (!qb.exec()) {
427  mLastError = QByteArrayLiteral("Unable to query parts.");
428  throw ItemRetrieverException(mLastError);
429  }
430 
431  const Part::List externalParts = qb.result();
432  for (Part part : externalParts) {
433  PartHelper::verify(part);
434  }
435 }
436 
437 QByteArray ItemRetriever::lastError() const
438 {
439  return mLastError;
440 }
qlonglong toLongLong(bool *ok) const const
bool verify(Part &part)
Verifies and if necessary fixes the external reference of this part.
Definition: parthelper.cpp:164
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
void addCondition(const Condition &condition)
Add a WHERE condition.
Definition: query.cpp:54
void setCollection(const Collection &collection, bool recursive=true)
Retrieve all items in the given collection.
void quit()
QHash::iterator insert(const Key &key, const T &value)
QSqlQuery & query()
Returns the query, only valid after exec().
Helper class for retrieving missing items parts from remote resources.
Definition: itemretriever.h:38
void itemSetToQuery(const ImapSet &set, QueryBuilder &qb, const Collection &collection=Collection())
Add conditions to qb for the given item set set.
Represents a collection of PIM items.
Definition: collection.h:61
QVector::iterator erase(QVector::iterator begin, QVector::iterator end)
bool startsWith(const QByteArray &ba) const const
NOTE: only supported for UPDATE and SELECT queries.
Definition: querybuilder.h:48
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
QSet::iterator insert(const T &value)
void addColumn(const QString &col)
Adds the given column to a select query.
QString fromUtf8(const char *str, int size)
bool isNull() const const
int exec(QEventLoop::ProcessEventsFlags flags)
QVariant value(int index) const const
Manages and processes item retrieval requests.
void addSortColumn(const QString &column, Query::SortOrder order=Query::Ascending)
Add sort column.
void exit(int returnCode)
bool next()
QVector< T > result()
Returns the result of this SELECT query.
void addColumnCondition(const QString &column, CompareOperator op, const QString &column2)
Add a WHERE condition which compares a column with another column.
Definition: query.cpp:22
void setScope(const Scope &scope)
Retrieve all items matching the given item scope.
NOTE: only supported for SELECT queries.
Definition: querybuilder.h:50
void push_back(char ch)
QList::iterator end()
QHash::iterator find(const Key &key)
bool contains(const T &value) const const
bool isValid() const const
Details of a single item retrieval request.
Helper class for creating and executing database SELECT queries.
void finish()
void insert(int i, const T &value)
QSet::iterator end()
Helper integration between Akonadi and Qt.
void push_back(const T &value)
Represents a WHERE condition tree.
Definition: query.h:61
QHash::iterator end()
void clear()
void scopeToQuery(const Scope &scope, const CommandContext &context, QueryBuilder &qb)
Add conditions to qb for the given item operation scope scope.
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:31
QList::iterator begin()
Query::Condition conditionFromFqNames(const QStringList &fqNames)
Returns a query condition that matches the given part type list.
bool exec()
Executes the query, returns true on success.
An Connection represents one connection of a client to the server.
Definition: connection.h:38
void addValueCondition(const QString &column, CompareOperator op, const QVariant &value)
Add a WHERE condition which compares a column with a given value.
Definition: query.cpp:12
This file is part of the KDE documentation.
Documentation copyright © 1996-2021 The KDE developers.
Generated on Thu Jul 29 2021 23:17:47 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.