Akonadi

itemfetchhelper.cpp
1 
2 /***************************************************************************
3  * SPDX-FileCopyrightText: 2006-2009 Tobias Koenig <[email protected]> *
4  * *
5  * SPDX-License-Identifier: LGPL-2.0-or-later *
6  ***************************************************************************/
7 
8 #include "itemfetchhelper.h"
9 
10 #include "akonadi.h"
11 #include "connection.h"
12 #include "handler.h"
13 #include "handlerhelper.h"
14 #include "storage/selectquerybuilder.h"
15 #include "storage/itemqueryhelper.h"
16 #include "storage/itemretrievalmanager.h"
17 #include "storage/itemretrievalrequest.h"
18 #include "storage/parthelper.h"
19 #include "storage/parttypehelper.h"
20 #include "storage/transaction.h"
21 #include "shared/akranges.h"
22 
23 #include "utils.h"
24 #include "intervalcheck.h"
25 #include "agentmanagerinterface.h"
26 #include "tagfetchhelper.h"
27 #include "relationfetchhandler.h"
28 #include "akonadiserver_debug.h"
29 
30 #include <private/scope_p.h>
31 #include <private/dbus_p.h>
32 
33 #include <QStringList>
34 #include <QVariant>
35 #include <QDateTime>
36 #include <QSqlQuery>
37 
38 #include <QElapsedTimer>
39 
40 using namespace Akonadi;
41 using namespace Akonadi::Server;
42 using namespace AkRanges;
43 
44 #define ENABLE_FETCH_PROFILING 0
45 #if ENABLE_FETCH_PROFILING
46 #define BEGIN_TIMER(name) \
47  QElapsedTimer name##Timer; \
48  name##Timer.start();
49 
50 #define END_TIMER(name) \
51  const double name##Elapsed = name##Timer.nsecsElapsed() / 1000000.0;
52 #define PROF_INC(name) \
53  ++name;
54 #else
55 #define BEGIN_TIMER(name)
56 #define END_TIMER(name)
57 #define PROF_INC(name)
58 #endif
59 
60 ItemFetchHelper::ItemFetchHelper(Connection *connection, const Scope &scope,
61  const Protocol::ItemFetchScope &itemFetchScope,
62  const Protocol::TagFetchScope &tagFetchScope,
63  AkonadiServer &akonadi)
64  : ItemFetchHelper(connection, connection->context(), scope, itemFetchScope, tagFetchScope, akonadi)
65 {
66 }
67 
68 
69 ItemFetchHelper::ItemFetchHelper(Connection *connection, const CommandContext &context,
70  const Scope &scope,
71  const Protocol::ItemFetchScope &itemFetchScope,
72  const Protocol::TagFetchScope &tagFetchScope,
73  AkonadiServer &akonadi)
74  : mConnection(connection)
75  , mContext(context)
76  , mScope(scope)
77  , mItemFetchScope(itemFetchScope)
78  , mTagFetchScope(tagFetchScope)
79  , mAkonadi(akonadi)
80 {
81  std::fill(mItemQueryColumnMap, mItemQueryColumnMap + ItemQueryColumnCount, -1);
82 }
83 
84 void ItemFetchHelper::disableATimeUpdates()
85 {
86  mUpdateATimeEnabled = false;
87 }
88 
89 enum PartQueryColumns {
90  PartQueryPimIdColumn,
91  PartQueryTypeIdColumn,
92  PartQueryDataColumn,
93  PartQueryStorageColumn,
94  PartQueryVersionColumn,
95  PartQueryDataSizeColumn
96 };
97 
98 QSqlQuery ItemFetchHelper::buildPartQuery(const QVector<QByteArray> &partList, bool allPayload, bool allAttrs)
99 {
101  QueryBuilder partQuery(PimItem::tableName());
102 
103  if (!partList.isEmpty() || allPayload || allAttrs) {
104  partQuery.addJoin(QueryBuilder::InnerJoin, Part::tableName(), PimItem::idFullColumnName(), Part::pimItemIdFullColumnName());
105  partQuery.addColumn(PimItem::idFullColumnName());
106  partQuery.addColumn(Part::partTypeIdFullColumnName());
107  partQuery.addColumn(Part::dataFullColumnName());
108  partQuery.addColumn(Part::storageFullColumnName());
109  partQuery.addColumn(Part::versionFullColumnName());
110  partQuery.addColumn(Part::datasizeFullColumnName());
111 
112  partQuery.addSortColumn(PimItem::idFullColumnName(), Query::Descending);
113 
114  if (!partList.isEmpty() || allPayload || allAttrs) {
115  Query::Condition cond(Query::Or);
116  for (const QByteArray &b : qAsConst(partList)) {
117  if (b.startsWith("PLD") || b.startsWith("ATR")) {
118  cond.addValueCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartTypeHelper::fromFqName(b).id());
119  }
120  }
121  if (allPayload || allAttrs) {
122  partQuery.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName());
123  if (allPayload) {
124  cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
125  }
126  if (allAttrs) {
127  cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("ATR"));
128  }
129  }
130 
131  partQuery.addCondition(cond);
132  }
133 
134  ItemQueryHelper::scopeToQuery(mScope, mContext, partQuery);
135 
136  if (!partQuery.exec()) {
137  throw HandlerException("Unable to list item parts");
138  }
139  partQuery.query().next();
140  }
141 
142  return partQuery.query();
143 }
144 
145 QSqlQuery ItemFetchHelper::buildItemQuery()
146 {
147  QueryBuilder itemQuery(PimItem::tableName());
148 
149  int column = 0;
150 #define ADD_COLUMN(colName, colId) { itemQuery.addColumn( colName ); mItemQueryColumnMap[colId] = column++; }
151  ADD_COLUMN(PimItem::idFullColumnName(), ItemQueryPimItemIdColumn);
152  if (mItemFetchScope.fetchRemoteId()) {
153  ADD_COLUMN(PimItem::remoteIdFullColumnName(), ItemQueryPimItemRidColumn)
154  }
155  ADD_COLUMN(PimItem::mimeTypeIdFullColumnName(), ItemQueryMimeTypeIdColumn)
156  ADD_COLUMN(PimItem::revFullColumnName(), ItemQueryRevColumn)
157  if (mItemFetchScope.fetchRemoteRevision()) {
158  ADD_COLUMN(PimItem::remoteRevisionFullColumnName(), ItemQueryRemoteRevisionColumn)
159  }
160  if (mItemFetchScope.fetchSize()) {
161  ADD_COLUMN(PimItem::sizeFullColumnName(), ItemQuerySizeColumn)
162  }
163  if (mItemFetchScope.fetchMTime()) {
164  ADD_COLUMN(PimItem::datetimeFullColumnName(), ItemQueryDatetimeColumn)
165  }
166  ADD_COLUMN(PimItem::collectionIdFullColumnName(), ItemQueryCollectionIdColumn)
167  if (mItemFetchScope.fetchGID()) {
168  ADD_COLUMN(PimItem::gidFullColumnName(), ItemQueryPimItemGidColumn)
169  }
170 #undef ADD_COLUMN
171 
172  itemQuery.addSortColumn(PimItem::idFullColumnName(), Query::Descending);
173 
174  ItemQueryHelper::scopeToQuery(mScope, mContext, itemQuery);
175 
176  if (mItemFetchScope.changedSince().isValid()) {
177  itemQuery.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual, mItemFetchScope.changedSince().toUTC());
178  }
179 
180  if (!itemQuery.exec()) {
181  throw HandlerException("Unable to list items");
182  }
183 
184  itemQuery.query().next();
185 
186  return itemQuery.query();
187 }
188 
189 enum FlagQueryColumns {
190  FlagQueryPimItemIdColumn,
191  FlagQueryFlagIdColumn
192 };
193 
194 QSqlQuery ItemFetchHelper::buildFlagQuery()
195 {
196  QueryBuilder flagQuery(PimItem::tableName());
197  flagQuery.addJoin(QueryBuilder::InnerJoin, PimItemFlagRelation::tableName(),
198  PimItem::idFullColumnName(), PimItemFlagRelation::leftFullColumnName());
199  flagQuery.addColumn(PimItem::idFullColumnName());
200  flagQuery.addColumn(PimItemFlagRelation::rightFullColumnName());
201 
202  ItemQueryHelper::scopeToQuery(mScope, mContext, flagQuery);
203  flagQuery.addSortColumn(PimItem::idFullColumnName(), Query::Descending);
204 
205  if (!flagQuery.exec()) {
206  throw HandlerException("Unable to retrieve item flags");
207  }
208 
209  flagQuery.query().next();
210 
211  return flagQuery.query();
212 }
213 
214 enum TagQueryColumns {
215  TagQueryItemIdColumn,
216  TagQueryTagIdColumn,
217 };
218 
219 QSqlQuery ItemFetchHelper::buildTagQuery()
220 {
221  QueryBuilder tagQuery(PimItem::tableName());
222  tagQuery.addJoin(QueryBuilder::InnerJoin, PimItemTagRelation::tableName(),
223  PimItem::idFullColumnName(), PimItemTagRelation::leftFullColumnName());
224  tagQuery.addJoin(QueryBuilder::InnerJoin, Tag::tableName(),
225  Tag::idFullColumnName(), PimItemTagRelation::rightFullColumnName());
226  tagQuery.addColumn(PimItem::idFullColumnName());
227  tagQuery.addColumn(Tag::idFullColumnName());
228 
229  ItemQueryHelper::scopeToQuery(mScope, mContext, tagQuery);
230  tagQuery.addSortColumn(PimItem::idFullColumnName(), Query::Descending);
231 
232  if (!tagQuery.exec()) {
233  throw HandlerException("Unable to retrieve item tags");
234  }
235 
236  tagQuery.query().next();
237 
238  return tagQuery.query();
239 }
240 
241 enum VRefQueryColumns {
242  VRefQueryCollectionIdColumn,
243  VRefQueryItemIdColumn
244 };
245 
246 QSqlQuery ItemFetchHelper::buildVRefQuery()
247 {
248  QueryBuilder vRefQuery(PimItem::tableName());
249  vRefQuery.addJoin(QueryBuilder::LeftJoin, CollectionPimItemRelation::tableName(),
250  CollectionPimItemRelation::rightFullColumnName(),
251  PimItem::idFullColumnName());
252  vRefQuery.addColumn(CollectionPimItemRelation::leftFullColumnName());
253  vRefQuery.addColumn(CollectionPimItemRelation::rightFullColumnName());
254  ItemQueryHelper::scopeToQuery(mScope, mContext, vRefQuery);
255  vRefQuery.addSortColumn(PimItem::idFullColumnName(), Query::Descending);
256 
257  if (!vRefQuery.exec()) {
258  throw HandlerException("Unable to retrieve virtual references");
259  }
260 
261  vRefQuery.query().next();
262 
263  return vRefQuery.query();
264 }
265 
266 bool ItemFetchHelper::isScopeLocal(const Scope &scope)
267 {
268  // The only agent allowed to override local scope is the Baloo Indexer
269  if (!mConnection->sessionId().startsWith("akonadi_indexing_agent")) {
270  return false;
271  }
272 
273  // Get list of all resources that own all items in the scope
274  QueryBuilder qb(PimItem::tableName(), QueryBuilder::Select);
275  qb.setDistinct(true);
276  qb.addColumn(Resource::nameFullColumnName());
277  qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(),
278  PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
279  qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(),
280  Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
281  ItemQueryHelper::scopeToQuery(scope, mContext, qb);
282  if (mContext.resource().isValid()) {
283  qb.addValueCondition(Resource::nameFullColumnName(), Query::NotEquals,
284  mContext.resource().name());
285  }
286 
287  if (!qb.exec()) {
288  throw HandlerException("Failed to query database");
289  return false;
290  }
291 
292  // If there is more than one resource, i.e. this is a fetch from multiple
293  // collections, then don't bother and just return FALSE. This case is aimed
294  // specifically on Baloo, which fetches items from each collection independently,
295  // so it will pass this check.
296  QSqlQuery query = qb.query();
297  if (query.size() != 1) {
298  return false;
299  }
300 
301  query.next();
302  const QString resourceName = query.value(0).toString();
303  query.finish();
304 
305  org::freedesktop::Akonadi::AgentManager manager(DBus::serviceName(DBus::Control),
306  QStringLiteral("/AgentManager"),
307  QDBusConnection::sessionBus());
308  const QString typeIdentifier = manager.agentInstanceType(resourceName);
309  const QVariantMap properties = manager.agentCustomProperties(typeIdentifier);
310  return properties.value(QStringLiteral("HasLocalStorage"), false).toBool();
311 }
312 
313 DataStore *ItemFetchHelper::storageBackend() const
314 {
315  if (mConnection) {
316  if (auto *store = mConnection->storageBackend()) {
317  return store;
318  }
319  }
320 
321  return DataStore::self();
322 }
323 
324 
325 bool ItemFetchHelper::fetchItems(std::function<void(Protocol::FetchItemsResponse &&)> &&itemCallback)
326 {
327  BEGIN_TIMER(fetch)
328 
329  // retrieve missing parts
330  // HACK: isScopeLocal() is a workaround for resources that have cache expiration
331  // because when the cache expires, Baloo is not able to content of the items. So
332  // we allow fetch of items that belong to local resources (like maildir) to ignore
333  // cacheOnly and retrieve missing parts from the resource. However ItemRetriever
334  // is painfully slow with many items and is generally designed to fetch a few
335  // messages, not all of them. In the long term, we need a better way to do this.
336  BEGIN_TIMER(itemRetriever)
337  BEGIN_TIMER(scopeLocal)
338 #if ENABLE_FETCH_PROFILING
339  double scopeLocalElapsed = 0;
340 #endif
341  if (!mItemFetchScope.cacheOnly() || isScopeLocal(mScope)) {
342 #if ENABLE_FETCH_PROFILING
343  scopeLocalElapsed = scopeLocalTimer.elapsed();
344 #endif
345 
346  // trigger a collection sync if configured to do so
347  triggerOnDemandFetch();
348 
349  // Prepare for a call to ItemRetriever::exec();
350  // From a resource perspective the only parts that can be fetched are payloads.
351  ItemRetriever retriever(mAkonadi.itemRetrievalManager(), mConnection, mContext);
352  retriever.setScope(mScope);
353  retriever.setRetrieveParts(mItemFetchScope.requestedPayloads());
354  retriever.setRetrieveFullPayload(mItemFetchScope.fullPayload());
355  retriever.setChangedSince(mItemFetchScope.changedSince());
356  if (!retriever.exec() && !mItemFetchScope.ignoreErrors()) { // There we go, retrieve the missing parts from the resource.
357  if (mContext.resource().isValid()) {
358  throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1, resource %2) : %3")
359  .arg(mContext.collectionId())
360  .arg(mContext.resource().id())
361  .arg(QString::fromLatin1(retriever.lastError())));
362  } else {
363  throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1) : %2")
364  .arg(mContext.collectionId())
365  .arg(QString::fromLatin1(retriever.lastError())));
366  }
367  }
368  }
369  END_TIMER(itemRetriever)
370 
371  BEGIN_TIMER(items)
372  QSqlQuery itemQuery = buildItemQuery();
373  END_TIMER(items)
374 
375  // error if query did not find any item and scope is not listing items but
376  // a request for a specific item
377  if (!itemQuery.isValid()) {
378  if (mItemFetchScope.ignoreErrors()) {
379  return true;
380  }
381  switch (mScope.scope()) {
382  case Scope::Uid: // fall through
383  case Scope::Rid: // fall through
384  case Scope::HierarchicalRid: // fall through
385  case Scope::Gid:
386  throw HandlerException("Item query returned empty result set");
387  break;
388  default:
389  break;
390  }
391  }
392  // build part query if needed
393  BEGIN_TIMER(parts)
394  QSqlQuery partQuery(storageBackend()->database());
395  if (!mItemFetchScope.requestedParts().isEmpty() || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
396  partQuery = buildPartQuery(mItemFetchScope.requestedParts(), mItemFetchScope.fullPayload(), mItemFetchScope.allAttributes());
397  }
398  END_TIMER(parts)
399 
400  // build flag query if needed
401  BEGIN_TIMER(flags)
402  QSqlQuery flagQuery(storageBackend()->database());
403  if (mItemFetchScope.fetchFlags()) {
404  flagQuery = buildFlagQuery();
405  }
406  END_TIMER(flags)
407 
408  // build tag query if needed
409  BEGIN_TIMER(tags)
410  QSqlQuery tagQuery(storageBackend()->database());
411  if (mItemFetchScope.fetchTags()) {
412  tagQuery = buildTagQuery();
413  }
414  END_TIMER(tags)
415 
416  BEGIN_TIMER(vRefs)
417  QSqlQuery vRefQuery(storageBackend()->database());
418  if (mItemFetchScope.fetchVirtualReferences()) {
419  vRefQuery = buildVRefQuery();
420  }
421  END_TIMER(vRefs)
422 
423 #if ENABLE_FETCH_PROFILING
424  int itemsCount = 0;
425  int flagsCount = 0;
426  int partsCount = 0;
427  int tagsCount = 0;
428  int vRefsCount = 0;
429 #endif
430 
431  BEGIN_TIMER(processing)
432  QHash<qint64, QByteArray> flagIdNameCache;
433  QHash<qint64, QString> mimeTypeIdNameCache;
434  QHash<qint64, QByteArray> partTypeIdNameCache;
435  while (itemQuery.isValid()) {
436  PROF_INC(itemsCount)
437 
438  const qint64 pimItemId = extractQueryResult(itemQuery, ItemQueryPimItemIdColumn).toLongLong();
439  const int pimItemRev = extractQueryResult(itemQuery, ItemQueryRevColumn).toInt();
440 
441  Protocol::FetchItemsResponse response;
442  response.setId(pimItemId);
443  response.setRevision(pimItemRev);
444  const qint64 mimeTypeId = extractQueryResult(itemQuery, ItemQueryMimeTypeIdColumn).toLongLong();
445  auto mtIter = mimeTypeIdNameCache.find(mimeTypeId);
446  if (mtIter == mimeTypeIdNameCache.end()) {
447  mtIter = mimeTypeIdNameCache.insert(mimeTypeId, MimeType::retrieveById(mimeTypeId).name());
448  }
449  response.setMimeType(mtIter.value());
450  if (mItemFetchScope.fetchRemoteId()) {
451  response.setRemoteId(extractQueryResult(itemQuery, ItemQueryPimItemRidColumn).toString());
452  }
453  response.setParentId(extractQueryResult(itemQuery, ItemQueryCollectionIdColumn).toLongLong());
454 
455  if (mItemFetchScope.fetchSize()) {
456  response.setSize(extractQueryResult(itemQuery, ItemQuerySizeColumn).toLongLong());
457  }
458  if (mItemFetchScope.fetchMTime()) {
459  response.setMTime(Utils::variantToDateTime(extractQueryResult(itemQuery, ItemQueryDatetimeColumn)));
460  }
461  if (mItemFetchScope.fetchRemoteRevision()) {
462  response.setRemoteRevision(extractQueryResult(itemQuery, ItemQueryRemoteRevisionColumn).toString());
463  }
464  if (mItemFetchScope.fetchGID()) {
465  response.setGid(extractQueryResult(itemQuery, ItemQueryPimItemGidColumn).toString());
466  }
467 
468  if (mItemFetchScope.fetchFlags()) {
469  QVector<QByteArray> flags;
470  while (flagQuery.isValid()) {
471  const qint64 id = flagQuery.value(FlagQueryPimItemIdColumn).toLongLong();
472  if (id > pimItemId) {
473  flagQuery.next();
474  continue;
475  } else if (id < pimItemId) {
476  break;
477  }
478  const qint64 flagId = flagQuery.value(FlagQueryFlagIdColumn).toLongLong();
479  auto flagNameIter = flagIdNameCache.find(flagId);
480  if (flagNameIter == flagIdNameCache.end()) {
481  flagNameIter = flagIdNameCache.insert(flagId, Flag::retrieveById(flagId).name().toUtf8());
482  }
483  flags << flagNameIter.value();
484  flagQuery.next();
485  }
486  response.setFlags(flags);
487  }
488 
489  if (mItemFetchScope.fetchTags()) {
490  QVector<qint64> tagIds;
491  QVector<Protocol::FetchTagsResponse> tags;
492  while (tagQuery.isValid()) {
493  PROF_INC(tagsCount)
494  const qint64 id = tagQuery.value(TagQueryItemIdColumn).toLongLong();
495  if (id > pimItemId) {
496  tagQuery.next();
497  continue;
498  } else if (id < pimItemId) {
499  break;
500  }
501  tagIds << tagQuery.value(TagQueryTagIdColumn).toLongLong();
502  tagQuery.next();
503  }
504 
505  if (mTagFetchScope.fetchIdOnly()) {
506  tags = tagIds | Views::transform([](const auto tagId) {
507  Protocol::FetchTagsResponse resp;
508  resp.setId(tagId);
509  return resp;
510  })
511  | Actions::toQVector;
512  } else {
513  tags = tagIds | Views::transform([this](const auto tagId) {
514  return HandlerHelper::fetchTagsResponse(Tag::retrieveById(tagId), mTagFetchScope, mConnection);
515  })
516  | Actions::toQVector;
517  }
518  response.setTags(tags);
519  }
520 
521  if (mItemFetchScope.fetchVirtualReferences()) {
522  QVector<qint64> vRefs;
523  while (vRefQuery.isValid()) {
524  PROF_INC(vRefsCount)
525  const qint64 id = vRefQuery.value(VRefQueryItemIdColumn).toLongLong();
526  if (id > pimItemId) {
527  vRefQuery.next();
528  continue;
529  } else if (id < pimItemId) {
530  break;
531  }
532  vRefs << vRefQuery.value(VRefQueryCollectionIdColumn).toLongLong();
533  vRefQuery.next();
534  }
535  response.setVirtualReferences(vRefs);
536  }
537 
538  if (mItemFetchScope.fetchRelations()) {
539  SelectQueryBuilder<Relation> qb;
540  Query::Condition condition;
541  condition.setSubQueryMode(Query::Or);
542  condition.addValueCondition(Relation::leftIdFullColumnName(), Query::Equals, pimItemId);
543  condition.addValueCondition(Relation::rightIdFullColumnName(), Query::Equals, pimItemId);
544  qb.addCondition(condition);
545  qb.addGroupColumns(QStringList() << Relation::leftIdColumn() << Relation::rightIdColumn() << Relation::typeIdColumn() << Relation::remoteIdColumn());
546  if (!qb.exec()) {
547  throw HandlerException("Unable to list item relations");
548  }
549  QVector<Protocol::FetchRelationsResponse> relations;
550  const auto result = qb.result();
551  relations.reserve(result.size());
552  for (const Relation &rel : result) {
553  relations.push_back(HandlerHelper::fetchRelationsResponse(rel));;
554  }
555  response.setRelations(relations);
556  }
557 
558  if (mItemFetchScope.ancestorDepth() != Protocol::ItemFetchScope::NoAncestor) {
559  response.setAncestors(ancestorsForItem(response.parentId()));
560  }
561 
562  bool skipItem = false;
563 
564  QVector<QByteArray> cachedParts;
565  QVector<Protocol::StreamPayloadResponse> parts;
566  while (partQuery.isValid()) {
567  PROF_INC(partsCount)
568  const qint64 id = partQuery.value(PartQueryPimIdColumn).toLongLong();
569  if (id > pimItemId) {
570  partQuery.next();
571  continue;
572  } else if (id < pimItemId) {
573  break;
574  }
575 
576  const qint64 partTypeId = partQuery.value(PartQueryTypeIdColumn).toLongLong();
577  auto ptIter = partTypeIdNameCache.find(partTypeId);
578  if (ptIter == partTypeIdNameCache.end()) {
579  ptIter = partTypeIdNameCache.insert(partTypeId, PartTypeHelper::fullName(PartType::retrieveById(partTypeId)).toUtf8());
580  }
581  Protocol::PartMetaData metaPart;
582  Protocol::StreamPayloadResponse partData;
583  partData.setPayloadName(ptIter.value());
584  metaPart.setName(ptIter.value());
585  metaPart.setVersion(partQuery.value(PartQueryVersionColumn).toInt());
586  metaPart.setSize(partQuery.value(PartQueryDataSizeColumn).toLongLong());
587 
588  const QByteArray data = Utils::variantToByteArray(partQuery.value(PartQueryDataColumn));
589  if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
590  if (!data.isEmpty()) {
591  cachedParts << ptIter.value();
592  }
593  partQuery.next();
594  } else {
595  if (mItemFetchScope.ignoreErrors() && data.isEmpty()) {
596  //We wanted the payload, couldn't get it, and are ignoring errors. Skip the item.
597  //This is not an error though, it's fine to have empty payload parts (to denote existing but not cached parts)
598  qCDebug(AKONADISERVER_LOG) << "item" << id << "has an empty payload part in parttable for part" << metaPart.name();
599  skipItem = true;
600  break;
601  }
602  metaPart.setStorageType(static_cast<Protocol::PartMetaData::StorageType>(
603  partQuery.value(PartQueryStorageColumn).toInt()));
604  if (data.isEmpty()) {
605  partData.setData(QByteArray(""));
606  } else {
607  partData.setData(data);
608  }
609  partData.setMetaData(metaPart);
610 
611  if (mItemFetchScope.requestedParts().contains(ptIter.value()) || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
612  parts.append(partData);
613  }
614 
615  partQuery.next();
616  }
617  }
618  response.setParts(parts);
619 
620  if (skipItem) {
621  itemQuery.next();
622  continue;
623  }
624 
625  if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
626  response.setCachedParts(cachedParts);
627  }
628 
629  if (itemCallback) {
630  itemCallback(std::move(response));
631  } else {
632  mConnection->sendResponse(std::move(response));
633  }
634 
635  itemQuery.next();
636  }
637  tagQuery.finish();
638  flagQuery.finish();
639  partQuery.finish();
640  vRefQuery.finish();
641  itemQuery.finish();
642  END_TIMER(processing)
643 
644  // update atime (only if the payload was actually requested, otherwise a simple resource sync prevents cache clearing)
645  BEGIN_TIMER(aTime)
646  if (mUpdateATimeEnabled &&
647  (needsAccessTimeUpdate(mItemFetchScope.requestedParts()) || mItemFetchScope.fullPayload())) {
648  updateItemAccessTime();
649  }
650  END_TIMER(aTime)
651 
652  END_TIMER(fetch)
653 #if ENABLE_FETCH_PROFILING
654  qCDebug(AKONADISERVER_LOG) << "ItemFetchHelper execution stats:";
655  qCDebug(AKONADISERVER_LOG) << "\tItems query:" << itemsElapsed << "ms," << itemsCount << " items in total";
656  qCDebug(AKONADISERVER_LOG) << "\tFlags query:" << flagsElapsed << "ms, " << flagsCount << " flags in total";
657  qCDebug(AKONADISERVER_LOG) << "\tParts query:" << partsElapsed << "ms, " << partsCount << " parts in total";
658  qCDebug(AKONADISERVER_LOG) << "\tTags query: " << tagsElapsed << "ms, " << tagsCount << " tags in total";
659  qCDebug(AKONADISERVER_LOG) << "\tVRefs query:" << vRefsElapsed << "ms, " << vRefsCount << " vRefs in total";
660  qCDebug(AKONADISERVER_LOG) << "\t------------";
661  qCDebug(AKONADISERVER_LOG) << "\tItem retriever:" << itemRetrieverElapsed << "ms (scope local:" << scopeLocalElapsed << "ms)";
662  qCDebug(AKONADISERVER_LOG) << "\tTotal query:" << (itemsElapsed + flagsElapsed + partsElapsed + tagsElapsed + vRefsElapsed) << "ms";
663  qCDebug(AKONADISERVER_LOG) << "\tTotal processing: " << processingElapsed << "ms";
664  qCDebug(AKONADISERVER_LOG) << "\tATime update:" << aTimeElapsed << "ms";
665  qCDebug(AKONADISERVER_LOG) << "\t============";
666  qCDebug(AKONADISERVER_LOG) << "\tTotal FETCH:" << fetchElapsed << "ms";
667  qCDebug(AKONADISERVER_LOG);
668  qCDebug(AKONADISERVER_LOG);
669 #endif
670 
671  return true;
672 }
673 
674 bool ItemFetchHelper::needsAccessTimeUpdate(const QVector<QByteArray> &parts)
675 {
676  // TODO technically we should compare the part list with the cache policy of
677  // the parent collection of the retrieved items, but that's kinda expensive
678  // Only updating the atime if the full payload was requested is a good
679  // approximation though.
680  return parts.contains(AKONADI_PARAM_PLD_RFC822);
681 }
682 
683 void ItemFetchHelper::updateItemAccessTime()
684 {
685  Transaction transaction(storageBackend(), QStringLiteral("update atime"));
686  QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
687  qb.setColumnValue(PimItem::atimeColumn(), QDateTime::currentDateTimeUtc());
688  ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
689 
690  if (!qb.exec()) {
691  qCWarning(AKONADISERVER_LOG) << "Unable to update item access time";
692  } else {
693  transaction.commit();
694  }
695 }
696 
697 void ItemFetchHelper::triggerOnDemandFetch()
698 {
699  if (mContext.collectionId() <= 0 || mItemFetchScope.cacheOnly()) {
700  return;
701  }
702 
703  Collection collection = mContext.collection();
704 
705  // HACK: don't trigger on-demand syncing if the resource is the one triggering it
706  if (mConnection->sessionId() == collection.resource().name().toLatin1()) {
707  return;
708  }
709 
710  storageBackend()->activeCachePolicy(collection);
711  if (!collection.cachePolicySyncOnDemand()) {
712  return;
713  }
714 
715  mConnection->akonadi().intervalChecker().requestCollectionSync(collection);
716 }
717 
718 QVector<Protocol::Ancestor> ItemFetchHelper::ancestorsForItem(Collection::Id parentColId)
719 {
720  if (mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::NoAncestor || parentColId == 0) {
721  return QVector<Protocol::Ancestor>();
722  }
723  const auto it = mAncestorCache.constFind(parentColId);
724  if (it != mAncestorCache.cend()) {
725  return *it;
726  }
727 
728  QVector<Protocol::Ancestor> ancestors;
729  Collection col = Collection::retrieveById(parentColId);
730  const int depthNum = mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::ParentAncestor ? 1 : INT_MAX;
731  for (int i = 0; i < depthNum; ++i) {
732  if (!col.isValid()) {
733  Protocol::Ancestor ancestor;
734  ancestor.setId(0);
735  ancestors << ancestor;
736  break;
737  }
738  Protocol::Ancestor ancestor;
739  ancestor.setId(col.id());
740  ancestor.setRemoteId(col.remoteId());
741  ancestors << ancestor;
742  col = col.parent();
743  }
744  mAncestorCache.insert(parentColId, ancestors);
745  return ancestors;
746 }
747 
748 QVariant ItemFetchHelper::extractQueryResult(const QSqlQuery &query,
749  ItemFetchHelper::ItemQueryColumns column) const
750 {
751  const int colId = mItemQueryColumnMap[column];
752  Q_ASSERT(colId >= 0);
753  return query.value(colId);
754 }
QSqlQuery::size
int size() const const
QHash::insert
QHash::iterator insert(const Key &key, const T &value)
Akonadi::Collection::isValid
bool isValid() const
Returns whether the collection is valid.
Definition: collection.cpp:124
Akonadi::Relation
An Akonadi Relation.
Definition: relation.h:39
KItinerary::LocationUtil::name
QString name(const QVariant &location)
QVector::append
void append(const T &value)
Akonadi::Server::QueryBuilder::setColumnValue
void setColumnValue(const QString &column, const QVariant &value)
Sets a column to the given value (only valid for INSERT and UPDATE queries).
Definition: querybuilder.cpp:576
Akonadi::Server::QueryCache::query
std::optional< QSqlQuery > query(const QString &queryStatement)
Returns the cached (and prepared) query for queryStatement.
Definition: querycache.cpp:95
Akonadi::Server::ItemRetriever
Helper class for retrieving missing items parts from remote resources.
Definition: itemretriever.h:40
Akonadi::Collection
Represents a collection of PIM items.
Definition: collection.h:63
Akonadi::Collection::Id
qint64 Id
Describes the unique id type.
Definition: collection.h:69
QByteArray::isEmpty
bool isEmpty() const const
QVector::cend
QVector::const_iterator cend() const const
Akonadi::Server::QueryBuilder::InnerJoin
NOTE: only supported for UPDATE and SELECT queries.
Definition: querybuilder.h:49
QDBusConnection::sessionBus
QDBusConnection sessionBus()
Akonadi::Server::PartTypeHelper::fromFqName
PartType fromFqName(const QString &fqName)
Retrieve (or create) PartType for the given fully qualified name.
Definition: parttypehelper.cpp:31
QVector::value
T value(int i) const const
Akonadi::Server::PartTypeHelper::fullName
QString fullName(const PartType &type)
Returns full part name.
Definition: parttypehelper.cpp:70
QVector::contains
bool contains(const T &value) const const
Akonadi::Server::Transaction
Helper class for DataStore transaction handling.
Definition: transaction.h:24
QSqlQuery::value
QVariant value(int index) const const
KStandardGuiItem::properties
KGuiItem properties()
QSqlQuery::next
bool next()
Akonadi::Server::SelectQueryBuilder::result
QVector< T > result()
Returns the result of this SELECT query.
Definition: selectquerybuilder.h:35
Akonadi::Server::ItemRetriever::setScope
void setScope(const Scope &scope)
Retrieve all items matching the given item scope.
Definition: itemretriever.cpp:108
Akonadi::Server::QueryBuilder::LeftJoin
NOTE: only supported for SELECT queries.
Definition: querybuilder.h:51
Akonadi::Server::DataStore::activeCachePolicy
virtual void activeCachePolicy(Collection &col)
Determines the active cache policy for this Collection.
Definition: datastore.cpp:1019
QHash::value
const T value(const Key &key) const const
QVector::reserve
void reserve(int size)
Akonadi::Server::DataStore::self
static DataStore * self()
Per thread singleton.
Definition: datastore.cpp:219
Akonadi::Server::SelectQueryBuilder
Helper class for creating and executing database SELECT queries.
Definition: selectquerybuilder.h:20
QString::toLatin1
QByteArray toLatin1() const const
Akonadi::Collection::id
Id id() const
Returns the unique identifier of the collection.
Definition: collection.cpp:99
QTest::toString
char * toString(const T &value)
Akonadi::Server::Query::Condition::setSubQueryMode
void setSubQueryMode(LogicOperator op)
Set how sub-conditions should be combined, default is And.
Definition: query.cpp:49
Akonadi::Server::QueryBuilder::addGroupColumns
void addGroupColumns(const QStringList &columns)
Add list of columns to GROUP BY.
Definition: querybuilder.cpp:570
QSqlQuery::finish
void finish()
QVector::isEmpty
bool isEmpty() const const
Akonadi
Helper integration between Akonadi and Qt.
Akonadi::Collection::remoteId
QString remoteId() const
Returns the remote id of the collection.
Definition: collection.cpp:109
QVector::push_back
void push_back(const T &value)
QString::fromLatin1
QString fromLatin1(const char *str, int size)
Akonadi::Server::Query::Condition
Represents a WHERE condition tree.
Definition: query.h:64
Akonadi::Collection::resource
QString resource() const
Returns the identifier of the resource owning the collection.
Definition: collection.cpp:305
Akonadi::Server::DataStore
This class handles all the database access.
Definition: datastore.h:95
Akonadi::Server::ItemQueryHelper::scopeToQuery
void scopeToQuery(const Scope &scope, const CommandContext &context, QueryBuilder &qb)
Add conditions to qb for the given item operation scope scope.
Definition: itemqueryhelper.cpp:107
QVariant::toString
QString toString() const const
Akonadi::Server::QueryBuilder
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:32
QDateTime::currentDateTimeUtc
QDateTime currentDateTimeUtc()
Akonadi::Server::QueryBuilder::addCondition
void addCondition(const Query::Condition &condition, ConditionType type=WhereCondition)
Add a WHERE condition.
Definition: querybuilder.cpp:553
QMap::value
const T value(const Key &key, const T &defaultValue) const const
Akonadi::Server::QueryBuilder::exec
bool exec()
Executes the query, returns true on success.
Definition: querybuilder.cpp:337
Akonadi::Server::Connection
An Connection represents one connection of a client to the server.
Definition: connection.h:40
Akonadi::Server::Query::Condition::addValueCondition
void addValueCondition(const QString &column, CompareOperator op, const QVariant &value)
Add a WHERE condition which compares a column with a given value.
Definition: query.cpp:12
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Sun Jul 12 2020 23:16:56 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.