Akonadi

itemfetchhelper.cpp
1 
2 /***************************************************************************
3  * SPDX-FileCopyrightText: 2006-2009 Tobias Koenig <[email protected]> *
4  * *
5  * SPDX-License-Identifier: LGPL-2.0-or-later *
6  ***************************************************************************/
7 
8 #include "itemfetchhelper.h"
9 
10 #include "akonadi.h"
11 #include "connection.h"
12 #include "handler.h"
13 #include "handlerhelper.h"
14 #include "shared/akranges.h"
15 #include "storage/itemqueryhelper.h"
16 #include "storage/itemretrievalmanager.h"
17 #include "storage/itemretrievalrequest.h"
18 #include "storage/parthelper.h"
19 #include "storage/parttypehelper.h"
20 #include "storage/selectquerybuilder.h"
21 #include "storage/transaction.h"
22 
23 #include "agentmanagerinterface.h"
24 #include "akonadiserver_debug.h"
25 #include "intervalcheck.h"
26 #include "relationfetchhandler.h"
27 #include "tagfetchhelper.h"
28 #include "utils.h"
29 
30 #include <private/dbus_p.h>
31 
32 #include <QDateTime>
33 #include <QSqlQuery>
34 #include <QStringList>
35 #include <QVariant>
36 
37 #include <QElapsedTimer>
38 
39 using namespace Akonadi;
40 using namespace Akonadi::Server;
41 using namespace AkRanges;
42 
43 #define ENABLE_FETCH_PROFILING 0
44 #if ENABLE_FETCH_PROFILING
45 #define BEGIN_TIMER(name) \
46  QElapsedTimer name##Timer; \
47  name##Timer.start();
48 
49 #define END_TIMER(name) const double name##Elapsed = name##Timer.nsecsElapsed() / 1000000.0;
50 #define PROF_INC(name) ++name;
51 #else
52 #define BEGIN_TIMER(name)
53 #define END_TIMER(name)
54 #define PROF_INC(name)
55 #endif
56 
57 ItemFetchHelper::ItemFetchHelper(Connection *connection,
58  const Scope &scope,
59  const Protocol::ItemFetchScope &itemFetchScope,
60  const Protocol::TagFetchScope &tagFetchScope,
61  AkonadiServer &akonadi,
62  const Protocol::FetchLimit &itemsLimit)
63  : ItemFetchHelper(connection, connection->context(), scope, itemFetchScope, tagFetchScope, akonadi, itemsLimit)
64 {
65 }
66 
67 ItemFetchHelper::ItemFetchHelper(Connection *connection,
68  const CommandContext &context,
69  const Scope &scope,
70  const Protocol::ItemFetchScope &itemFetchScope,
71  const Protocol::TagFetchScope &tagFetchScope,
72  AkonadiServer &akonadi,
73  const Protocol::FetchLimit &itemsLimit)
74  : mConnection(connection)
75  , mContext(context)
76  , mScope(scope)
77  , mItemFetchScope(itemFetchScope)
78  , mTagFetchScope(tagFetchScope)
79  , mAkonadi(akonadi)
80  , mItemsLimit(itemsLimit)
81  , mItemQuery(PimItem::tableName())
82  , mPimItemQueryAlias(QLatin1String("pimItem_alias"))
83 {
84  std::fill(mItemQueryColumnMap, mItemQueryColumnMap + ItemQueryColumnCount, -1);
85 }
86 
87 void ItemFetchHelper::disableATimeUpdates()
88 {
89  mUpdateATimeEnabled = false;
90 }
91 
92 enum PartQueryColumns {
93  PartQueryPimIdColumn,
94  PartQueryTypeIdColumn,
95  PartQueryDataColumn,
96  PartQueryStorageColumn,
97  PartQueryVersionColumn,
98  PartQueryDataSizeColumn
99 };
100 
101 QSqlQuery ItemFetchHelper::buildPartQuery(const QVector<QByteArray> &partList, bool allPayload, bool allAttrs)
102 {
103  /// TODO: merge with ItemQuery
104  QueryBuilder partQuery(PimItem::tableName());
105  if (mItemsLimit.limit() > 0) {
106  partQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
107  }
108 
109  if (!partList.isEmpty() || allPayload || allAttrs) {
110  partQuery.addJoin(QueryBuilder::InnerJoin, Part::tableName(), partQuery.getTableWithColumn(PimItem::idColumn()), Part::pimItemIdFullColumnName());
111  partQuery.addColumn(partQuery.getTableWithColumn(PimItem::idColumn()));
112  partQuery.addColumn(Part::partTypeIdFullColumnName());
113  partQuery.addColumn(Part::dataFullColumnName());
114  partQuery.addColumn(Part::storageFullColumnName());
115  partQuery.addColumn(Part::versionFullColumnName());
116  partQuery.addColumn(Part::datasizeFullColumnName());
117 
118  partQuery.addSortColumn(partQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
119 
120  if (!partList.isEmpty() || allPayload || allAttrs) {
121  Query::Condition cond(Query::Or);
122  for (const QByteArray &b : std::as_const(partList)) {
123  if (b.startsWith("PLD") || b.startsWith("ATR")) {
124  cond.addValueCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartTypeHelper::fromFqName(b).id());
125  }
126  }
127  if (allPayload || allAttrs) {
128  partQuery.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName());
129  if (allPayload) {
130  cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
131  }
132  if (allAttrs) {
133  cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("ATR"));
134  }
135  }
136 
137  partQuery.addCondition(cond);
138  }
139 
140  ItemQueryHelper::scopeToQuery(mScope, mContext, partQuery);
141 
142  if (!partQuery.exec()) {
143  throw HandlerException("Unable to list item parts");
144  }
145  partQuery.query().next();
146  }
147 
148  return partQuery.query();
149 }
150 
151 QSqlQuery ItemFetchHelper::buildItemQuery()
152 {
153  int column = 0;
154 #define ADD_COLUMN(colName, colId) \
155  { \
156  mItemQuery.addColumn(colName); \
157  mItemQueryColumnMap[colId] = column++; \
158  }
159  ADD_COLUMN(PimItem::idFullColumnName(), ItemQueryPimItemIdColumn);
160  if (mItemFetchScope.fetchRemoteId()) {
161  ADD_COLUMN(PimItem::remoteIdFullColumnName(), ItemQueryPimItemRidColumn)
162  }
163  ADD_COLUMN(PimItem::mimeTypeIdFullColumnName(), ItemQueryMimeTypeIdColumn)
164  ADD_COLUMN(PimItem::revFullColumnName(), ItemQueryRevColumn)
165  if (mItemFetchScope.fetchRemoteRevision()) {
166  ADD_COLUMN(PimItem::remoteRevisionFullColumnName(), ItemQueryRemoteRevisionColumn)
167  }
168  if (mItemFetchScope.fetchSize()) {
169  ADD_COLUMN(PimItem::sizeFullColumnName(), ItemQuerySizeColumn)
170  }
171  if (mItemFetchScope.fetchMTime()) {
172  ADD_COLUMN(PimItem::datetimeFullColumnName(), ItemQueryDatetimeColumn)
173  }
174  ADD_COLUMN(PimItem::collectionIdFullColumnName(), ItemQueryCollectionIdColumn)
175  if (mItemFetchScope.fetchGID()) {
176  ADD_COLUMN(PimItem::gidFullColumnName(), ItemQueryPimItemGidColumn)
177  }
178 #undef ADD_COLUMN
179 
180  mItemQuery.addSortColumn(PimItem::idFullColumnName(), static_cast<Query::SortOrder>(mItemsLimit.sortOrder()));
181  if (mItemsLimit.limit() > 0) {
182  mItemQuery.setLimit(mItemsLimit.limit(), mItemsLimit.limitOffset());
183  }
184 
185  ItemQueryHelper::scopeToQuery(mScope, mContext, mItemQuery);
186 
187  if (mItemFetchScope.changedSince().isValid()) {
188  mItemQuery.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual, mItemFetchScope.changedSince().toUTC());
189  }
190 
191  if (!mItemQuery.exec()) {
192  throw HandlerException("Unable to list items");
193  }
194 
195  mItemQuery.query().next();
196 
197  return mItemQuery.query();
198 }
199 
200 enum FlagQueryColumns {
201  FlagQueryPimItemIdColumn,
202  FlagQueryFlagIdColumn,
203 };
204 
205 QSqlQuery ItemFetchHelper::buildFlagQuery()
206 {
207  QueryBuilder flagQuery(PimItem::tableName());
208  if (mItemsLimit.limit() > 0) {
209  flagQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
210  }
211 
212  flagQuery.addJoin(QueryBuilder::InnerJoin, PimItemFlagRelation::tableName(), flagQuery.getTableWithColumn(PimItem::idColumn()), PimItemFlagRelation::leftFullColumnName());
213 
214  flagQuery.addColumn(flagQuery.getTableWithColumn(PimItem::idColumn()));
215  flagQuery.addColumn(PimItemFlagRelation::rightFullColumnName());
216 
217  ItemQueryHelper::scopeToQuery(mScope, mContext, flagQuery);
218  flagQuery.addSortColumn(flagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
219 
220  if (!flagQuery.exec()) {
221  throw HandlerException("Unable to retrieve item flags");
222  }
223 
224  flagQuery.query().next();
225 
226  return flagQuery.query();
227 }
228 
229 enum TagQueryColumns {
230  TagQueryItemIdColumn,
231  TagQueryTagIdColumn,
232 };
233 
234 QSqlQuery ItemFetchHelper::buildTagQuery()
235 {
236  QueryBuilder tagQuery(PimItem::tableName());
237  if (mItemsLimit.limit() > 0) {
238  tagQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
239  }
240 
241  tagQuery.addJoin(QueryBuilder::InnerJoin, PimItemTagRelation::tableName(), tagQuery.getTableWithColumn(PimItem::idColumn()), PimItemTagRelation::leftFullColumnName());
242  tagQuery.addJoin(QueryBuilder::InnerJoin, Tag::tableName(), Tag::idFullColumnName(), PimItemTagRelation::rightFullColumnName());
243  tagQuery.addColumn(tagQuery.getTableWithColumn(PimItem::idColumn()));
244  tagQuery.addColumn(Tag::idFullColumnName());
245 
246  ItemQueryHelper::scopeToQuery(mScope, mContext, tagQuery);
247  tagQuery.addSortColumn(tagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
248 
249  if (!tagQuery.exec()) {
250  throw HandlerException("Unable to retrieve item tags");
251  }
252 
253  tagQuery.query().next();
254 
255  return tagQuery.query();
256 }
257 
258 enum VRefQueryColumns {
259  VRefQueryCollectionIdColumn,
260  VRefQueryItemIdColumn,
261 };
262 
263 QSqlQuery ItemFetchHelper::buildVRefQuery()
264 {
265  QueryBuilder vRefQuery(PimItem::tableName());
266  if(mItemsLimit.limit() > 0) {
267  vRefQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
268  }
269 
270  vRefQuery.addJoin(QueryBuilder::LeftJoin,
271  CollectionPimItemRelation::tableName(),
272  CollectionPimItemRelation::rightFullColumnName(),
273  vRefQuery.getTableWithColumn(PimItem::idColumn()));
274  vRefQuery.addColumn(CollectionPimItemRelation::leftFullColumnName());
275  vRefQuery.addColumn(CollectionPimItemRelation::rightFullColumnName());
276  ItemQueryHelper::scopeToQuery(mScope, mContext, vRefQuery);
277  vRefQuery.addSortColumn(vRefQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
278 
279  if (!vRefQuery.exec()) {
280  throw HandlerException("Unable to retrieve virtual references");
281  }
282 
283  vRefQuery.query().next();
284 
285  return vRefQuery.query();
286 }
287 
288 bool ItemFetchHelper::isScopeLocal(const Scope &scope)
289 {
290  // The only agent allowed to override local scope is the Baloo Indexer
291  if (!mConnection->sessionId().startsWith("akonadi_indexing_agent")) {
292  return false;
293  }
294 
295  // Get list of all resources that own all items in the scope
296  QueryBuilder qb(PimItem::tableName(), QueryBuilder::Select);
297  qb.setDistinct(true);
298  qb.addColumn(Resource::nameFullColumnName());
299  qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
300  qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
301  ItemQueryHelper::scopeToQuery(scope, mContext, qb);
302  if (mContext.resource().isValid()) {
303  qb.addValueCondition(Resource::nameFullColumnName(), Query::NotEquals, mContext.resource().name());
304  }
305 
306  if (!qb.exec()) {
307  throw HandlerException("Failed to query database");
308  return false;
309  }
310 
311  // If there is more than one resource, i.e. this is a fetch from multiple
312  // collections, then don't bother and just return FALSE. This case is aimed
313  // specifically on Baloo, which fetches items from each collection independently,
314  // so it will pass this check.
315  QSqlQuery query = qb.query();
316  if (query.size() != 1) {
317  return false;
318  }
319 
320  query.next();
321  const QString resourceName = query.value(0).toString();
322  query.finish();
323 
324  org::freedesktop::Akonadi::AgentManager manager(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus());
325  const QString typeIdentifier = manager.agentInstanceType(resourceName);
326  const QVariantMap properties = manager.agentCustomProperties(typeIdentifier);
327  return properties.value(QStringLiteral("HasLocalStorage"), false).toBool();
328 }
329 
330 DataStore *ItemFetchHelper::storageBackend() const
331 {
332  if (mConnection) {
333  if (auto store = mConnection->storageBackend()) {
334  return store;
335  }
336  }
337 
338  return DataStore::self();
339 }
340 
341 bool ItemFetchHelper::fetchItems(std::function<void(Protocol::FetchItemsResponse &&)> &&itemCallback)
342 {
343  BEGIN_TIMER(fetch)
344 
345  // retrieve missing parts
346  // HACK: isScopeLocal() is a workaround for resources that have cache expiration
347  // because when the cache expires, Baloo is not able to content of the items. So
348  // we allow fetch of items that belong to local resources (like maildir) to ignore
349  // cacheOnly and retrieve missing parts from the resource. However ItemRetriever
350  // is painfully slow with many items and is generally designed to fetch a few
351  // messages, not all of them. In the long term, we need a better way to do this.
352  BEGIN_TIMER(itemRetriever)
353  BEGIN_TIMER(scopeLocal)
354 #if ENABLE_FETCH_PROFILING
355  double scopeLocalElapsed = 0;
356 #endif
357  if (!mItemFetchScope.cacheOnly() || isScopeLocal(mScope)) {
358 #if ENABLE_FETCH_PROFILING
359  scopeLocalElapsed = scopeLocalTimer.elapsed();
360 #endif
361 
362  // trigger a collection sync if configured to do so
363  triggerOnDemandFetch();
364 
365  // Prepare for a call to ItemRetriever::exec();
366  // From a resource perspective the only parts that can be fetched are payloads.
367  ItemRetriever retriever(mAkonadi.itemRetrievalManager(), mConnection, mContext);
368  retriever.setScope(mScope);
369  retriever.setRetrieveParts(mItemFetchScope.requestedPayloads());
370  retriever.setRetrieveFullPayload(mItemFetchScope.fullPayload());
371  retriever.setChangedSince(mItemFetchScope.changedSince());
372  if (!retriever.exec() && !mItemFetchScope.ignoreErrors()) { // There we go, retrieve the missing parts from the resource.
373  if (mContext.resource().isValid()) {
374  throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1, resource %2) : %3")
375  .arg(mContext.collectionId())
376  .arg(mContext.resource().id())
377  .arg(QString::fromLatin1(retriever.lastError())));
378  } else {
379  throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1) : %2")
380  .arg(mContext.collectionId())
381  .arg(QString::fromLatin1(retriever.lastError())));
382  }
383  }
384  }
385  END_TIMER(itemRetriever)
386 
387  BEGIN_TIMER(items)
388  QSqlQuery itemQuery = buildItemQuery();
389  END_TIMER(items)
390 
391  // error if query did not find any item and scope is not listing items but
392  // a request for a specific item
393  if (!itemQuery.isValid()) {
394  if (mItemFetchScope.ignoreErrors()) {
395  return true;
396  }
397  switch (mScope.scope()) {
398  case Scope::Uid: // fall through
399  case Scope::Rid: // fall through
400  case Scope::HierarchicalRid: // fall through
401  case Scope::Gid:
402  throw HandlerException("Item query returned empty result set");
403  break;
404  default:
405  break;
406  }
407  }
408  // build part query if needed
409  BEGIN_TIMER(parts)
410  QSqlQuery partQuery(storageBackend()->database());
411  if (!mItemFetchScope.requestedParts().isEmpty() || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
412  partQuery = buildPartQuery(mItemFetchScope.requestedParts(), mItemFetchScope.fullPayload(), mItemFetchScope.allAttributes());
413  }
414  END_TIMER(parts)
415 
416  // build flag query if needed
417  BEGIN_TIMER(flags)
418  QSqlQuery flagQuery(storageBackend()->database());
419  if (mItemFetchScope.fetchFlags()) {
420  flagQuery = buildFlagQuery();
421  }
422  END_TIMER(flags)
423 
424  // build tag query if needed
425  BEGIN_TIMER(tags)
426  QSqlQuery tagQuery(storageBackend()->database());
427  if (mItemFetchScope.fetchTags()) {
428  tagQuery = buildTagQuery();
429  }
430  END_TIMER(tags)
431 
432  BEGIN_TIMER(vRefs)
433  QSqlQuery vRefQuery(storageBackend()->database());
434  if (mItemFetchScope.fetchVirtualReferences()) {
435  vRefQuery = buildVRefQuery();
436  }
437  END_TIMER(vRefs)
438 
439 #if ENABLE_FETCH_PROFILING
440  int itemsCount = 0;
441  int flagsCount = 0;
442  int partsCount = 0;
443  int tagsCount = 0;
444  int vRefsCount = 0;
445 #endif
446 
447  BEGIN_TIMER(processing)
448  QHash<qint64, QByteArray> flagIdNameCache;
449  QHash<qint64, QString> mimeTypeIdNameCache;
450  QHash<qint64, QByteArray> partTypeIdNameCache;
451  while (itemQuery.isValid()) {
452  PROF_INC(itemsCount)
453 
454  const qint64 pimItemId = extractQueryResult(itemQuery, ItemQueryPimItemIdColumn).toLongLong();
455  const int pimItemRev = extractQueryResult(itemQuery, ItemQueryRevColumn).toInt();
456 
457  Protocol::FetchItemsResponse response;
458  response.setId(pimItemId);
459  response.setRevision(pimItemRev);
460  const qint64 mimeTypeId = extractQueryResult(itemQuery, ItemQueryMimeTypeIdColumn).toLongLong();
461  auto mtIter = mimeTypeIdNameCache.find(mimeTypeId);
462  if (mtIter == mimeTypeIdNameCache.end()) {
463  mtIter = mimeTypeIdNameCache.insert(mimeTypeId, MimeType::retrieveById(mimeTypeId).name());
464  }
465  response.setMimeType(mtIter.value());
466  if (mItemFetchScope.fetchRemoteId()) {
467  response.setRemoteId(extractQueryResult(itemQuery, ItemQueryPimItemRidColumn).toString());
468  }
469  response.setParentId(extractQueryResult(itemQuery, ItemQueryCollectionIdColumn).toLongLong());
470 
471  if (mItemFetchScope.fetchSize()) {
472  response.setSize(extractQueryResult(itemQuery, ItemQuerySizeColumn).toLongLong());
473  }
474  if (mItemFetchScope.fetchMTime()) {
475  response.setMTime(Utils::variantToDateTime(extractQueryResult(itemQuery, ItemQueryDatetimeColumn)));
476  }
477  if (mItemFetchScope.fetchRemoteRevision()) {
478  response.setRemoteRevision(extractQueryResult(itemQuery, ItemQueryRemoteRevisionColumn).toString());
479  }
480  if (mItemFetchScope.fetchGID()) {
481  response.setGid(extractQueryResult(itemQuery, ItemQueryPimItemGidColumn).toString());
482  }
483 
484  if (mItemFetchScope.fetchFlags()) {
485  QVector<QByteArray> flags;
486  while (flagQuery.isValid()) {
487  const qint64 id = flagQuery.value(FlagQueryPimItemIdColumn).toLongLong();
488  if (id > pimItemId) {
489  flagQuery.next();
490  continue;
491  } else if (id < pimItemId) {
492  break;
493  }
494  const qint64 flagId = flagQuery.value(FlagQueryFlagIdColumn).toLongLong();
495  auto flagNameIter = flagIdNameCache.find(flagId);
496  if (flagNameIter == flagIdNameCache.end()) {
497  flagNameIter = flagIdNameCache.insert(flagId, Flag::retrieveById(flagId).name().toUtf8());
498  }
499  flags << flagNameIter.value();
500  flagQuery.next();
501  }
502  response.setFlags(flags);
503  }
504 
505  if (mItemFetchScope.fetchTags()) {
506  QVector<qint64> tagIds;
508  while (tagQuery.isValid()) {
509  PROF_INC(tagsCount)
510  const qint64 id = tagQuery.value(TagQueryItemIdColumn).toLongLong();
511  if (id > pimItemId) {
512  tagQuery.next();
513  continue;
514  } else if (id < pimItemId) {
515  break;
516  }
517  tagIds << tagQuery.value(TagQueryTagIdColumn).toLongLong();
518  tagQuery.next();
519  }
520 
521  if (mTagFetchScope.fetchIdOnly()) {
522  tags = tagIds | Views::transform([](const auto tagId) {
523  Protocol::FetchTagsResponse resp;
524  resp.setId(tagId);
525  return resp;
526  })
527  | Actions::toQVector;
528  } else {
529  tags = tagIds | Views::transform([this](const auto tagId) {
530  return HandlerHelper::fetchTagsResponse(Tag::retrieveById(tagId), mTagFetchScope, mConnection);
531  })
532  | Actions::toQVector;
533  }
534  response.setTags(tags);
535  }
536 
537  if (mItemFetchScope.fetchVirtualReferences()) {
538  QVector<qint64> vRefs;
539  while (vRefQuery.isValid()) {
540  PROF_INC(vRefsCount)
541  const qint64 id = vRefQuery.value(VRefQueryItemIdColumn).toLongLong();
542  if (id > pimItemId) {
543  vRefQuery.next();
544  continue;
545  } else if (id < pimItemId) {
546  break;
547  }
548  vRefs << vRefQuery.value(VRefQueryCollectionIdColumn).toLongLong();
549  vRefQuery.next();
550  }
551  response.setVirtualReferences(vRefs);
552  }
553 
554  if (mItemFetchScope.fetchRelations()) {
556  Query::Condition condition;
557  condition.setSubQueryMode(Query::Or);
558  condition.addValueCondition(Relation::leftIdFullColumnName(), Query::Equals, pimItemId);
559  condition.addValueCondition(Relation::rightIdFullColumnName(), Query::Equals, pimItemId);
560  qb.addCondition(condition);
561  qb.addGroupColumns(QStringList() << Relation::leftIdColumn() << Relation::rightIdColumn() << Relation::typeIdColumn()
562  << Relation::remoteIdColumn());
563  if (!qb.exec()) {
564  throw HandlerException("Unable to list item relations");
565  }
567  const auto result = qb.result();
568  relations.reserve(result.size());
569  for (const Relation &rel : result) {
570  relations.push_back(HandlerHelper::fetchRelationsResponse(rel));
571  ;
572  }
573  response.setRelations(relations);
574  }
575 
576  if (mItemFetchScope.ancestorDepth() != Protocol::ItemFetchScope::NoAncestor) {
577  response.setAncestors(ancestorsForItem(response.parentId()));
578  }
579 
580  bool skipItem = false;
581 
582  QVector<QByteArray> cachedParts;
584  while (partQuery.isValid()) {
585  PROF_INC(partsCount)
586  const qint64 id = partQuery.value(PartQueryPimIdColumn).toLongLong();
587  if (id > pimItemId) {
588  partQuery.next();
589  continue;
590  } else if (id < pimItemId) {
591  break;
592  }
593 
594  const qint64 partTypeId = partQuery.value(PartQueryTypeIdColumn).toLongLong();
595  auto ptIter = partTypeIdNameCache.find(partTypeId);
596  if (ptIter == partTypeIdNameCache.end()) {
597  ptIter = partTypeIdNameCache.insert(partTypeId, PartTypeHelper::fullName(PartType::retrieveById(partTypeId)).toUtf8());
598  }
599  Protocol::PartMetaData metaPart;
600  Protocol::StreamPayloadResponse partData;
601  partData.setPayloadName(ptIter.value());
602  metaPart.setName(ptIter.value());
603  metaPart.setVersion(partQuery.value(PartQueryVersionColumn).toInt());
604  metaPart.setSize(partQuery.value(PartQueryDataSizeColumn).toLongLong());
605 
606  const QByteArray data = Utils::variantToByteArray(partQuery.value(PartQueryDataColumn));
607  if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
608  if (!data.isEmpty()) {
609  cachedParts << ptIter.value();
610  }
611  partQuery.next();
612  } else {
613  if (mItemFetchScope.ignoreErrors() && data.isEmpty()) {
614  // We wanted the payload, couldn't get it, and are ignoring errors. Skip the item.
615  // This is not an error though, it's fine to have empty payload parts (to denote existing but not cached parts)
616  qCDebug(AKONADISERVER_LOG) << "item" << id << "has an empty payload part in parttable for part" << metaPart.name();
617  skipItem = true;
618  break;
619  }
620  metaPart.setStorageType(static_cast<Protocol::PartMetaData::StorageType>(partQuery.value(PartQueryStorageColumn).toInt()));
621  if (data.isEmpty()) {
622  partData.setData(QByteArray(""));
623  } else {
624  partData.setData(data);
625  }
626  partData.setMetaData(metaPart);
627 
628  if (mItemFetchScope.requestedParts().contains(ptIter.value()) || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
629  parts.append(partData);
630  }
631 
632  partQuery.next();
633  }
634  }
635  response.setParts(parts);
636 
637  if (skipItem) {
638  itemQuery.next();
639  continue;
640  }
641 
642  if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
643  response.setCachedParts(cachedParts);
644  }
645 
646  if (itemCallback) {
647  itemCallback(std::move(response));
648  } else {
649  mConnection->sendResponse(std::move(response));
650  }
651 
652  itemQuery.next();
653  }
654  tagQuery.finish();
655  flagQuery.finish();
656  partQuery.finish();
657  vRefQuery.finish();
658  itemQuery.finish();
659  END_TIMER(processing)
660 
661  // update atime (only if the payload was actually requested, otherwise a simple resource sync prevents cache clearing)
662  BEGIN_TIMER(aTime)
663  if (mUpdateATimeEnabled && (needsAccessTimeUpdate(mItemFetchScope.requestedParts()) || mItemFetchScope.fullPayload())) {
664  updateItemAccessTime();
665  }
666  END_TIMER(aTime)
667 
668  END_TIMER(fetch)
669 #if ENABLE_FETCH_PROFILING
670  qCDebug(AKONADISERVER_LOG) << "ItemFetchHelper execution stats:";
671  qCDebug(AKONADISERVER_LOG) << "\tItems query:" << itemsElapsed << "ms," << itemsCount << " items in total";
672  qCDebug(AKONADISERVER_LOG) << "\tFlags query:" << flagsElapsed << "ms, " << flagsCount << " flags in total";
673  qCDebug(AKONADISERVER_LOG) << "\tParts query:" << partsElapsed << "ms, " << partsCount << " parts in total";
674  qCDebug(AKONADISERVER_LOG) << "\tTags query: " << tagsElapsed << "ms, " << tagsCount << " tags in total";
675  qCDebug(AKONADISERVER_LOG) << "\tVRefs query:" << vRefsElapsed << "ms, " << vRefsCount << " vRefs in total";
676  qCDebug(AKONADISERVER_LOG) << "\t------------";
677  qCDebug(AKONADISERVER_LOG) << "\tItem retriever:" << itemRetrieverElapsed << "ms (scope local:" << scopeLocalElapsed << "ms)";
678  qCDebug(AKONADISERVER_LOG) << "\tTotal query:" << (itemsElapsed + flagsElapsed + partsElapsed + tagsElapsed + vRefsElapsed) << "ms";
679  qCDebug(AKONADISERVER_LOG) << "\tTotal processing: " << processingElapsed << "ms";
680  qCDebug(AKONADISERVER_LOG) << "\tATime update:" << aTimeElapsed << "ms";
681  qCDebug(AKONADISERVER_LOG) << "\t============";
682  qCDebug(AKONADISERVER_LOG) << "\tTotal FETCH:" << fetchElapsed << "ms";
683  qCDebug(AKONADISERVER_LOG);
684  qCDebug(AKONADISERVER_LOG);
685 #endif
686 
687  return true;
688 }
689 
690 bool ItemFetchHelper::needsAccessTimeUpdate(const QVector<QByteArray> &parts)
691 {
692  // TODO technically we should compare the part list with the cache policy of
693  // the parent collection of the retrieved items, but that's kinda expensive
694  // Only updating the atime if the full payload was requested is a good
695  // approximation though.
696  return parts.contains(AKONADI_PARAM_PLD_RFC822);
697 }
698 
699 void ItemFetchHelper::updateItemAccessTime()
700 {
701  Transaction transaction(storageBackend(), QStringLiteral("update atime"));
702  QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
703  qb.setColumnValue(PimItem::atimeColumn(), QDateTime::currentDateTimeUtc());
704  ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
705 
706  if (!qb.exec()) {
707  qCWarning(AKONADISERVER_LOG) << "Unable to update item access time";
708  } else {
709  transaction.commit();
710  }
711 }
712 
713 void ItemFetchHelper::triggerOnDemandFetch()
714 {
715  if (mContext.collectionId() <= 0 || mItemFetchScope.cacheOnly()) {
716  return;
717  }
718 
719  Collection collection = mContext.collection();
720 
721  // HACK: don't trigger on-demand syncing if the resource is the one triggering it
722  if (mConnection->sessionId() == collection.resource().name().toLatin1()) {
723  return;
724  }
725 
726  storageBackend()->activeCachePolicy(collection);
727  if (!collection.cachePolicySyncOnDemand()) {
728  return;
729  }
730 
731  mConnection->akonadi().intervalChecker().requestCollectionSync(collection);
732 }
733 
734 QVector<Protocol::Ancestor> ItemFetchHelper::ancestorsForItem(Collection::Id parentColId)
735 {
736  if (mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::NoAncestor || parentColId == 0) {
738  }
739  const auto it = mAncestorCache.constFind(parentColId);
740  if (it != mAncestorCache.cend()) {
741  return *it;
742  }
743 
744  QVector<Protocol::Ancestor> ancestors;
745  Collection col = Collection::retrieveById(parentColId);
746  const int depthNum = mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::ParentAncestor ? 1 : INT_MAX;
747  for (int i = 0; i < depthNum; ++i) {
748  if (!col.isValid()) {
749  Protocol::Ancestor ancestor;
750  ancestor.setId(0);
751  ancestors << ancestor;
752  break;
753  }
754  Protocol::Ancestor ancestor;
755  ancestor.setId(col.id());
756  ancestor.setRemoteId(col.remoteId());
757  ancestors << ancestor;
758  col = col.parent();
759  }
760  mAncestorCache.insert(parentColId, ancestors);
761  return ancestors;
762 }
763 
764 QVariant ItemFetchHelper::extractQueryResult(const QSqlQuery &query, ItemFetchHelper::ItemQueryColumns column) const
765 {
766  const int colId = mItemQueryColumnMap[column];
767  Q_ASSERT(colId >= 0);
768  return query.value(colId);
769 }
This class handles all the database access.
Definition: datastore.h:94
bool isEmpty() const const
Helper class for DataStore transaction handling.
Definition: transaction.h:22
void scopeToQuery(const Scope &scope, const CommandContext &context, QueryBuilder &qb)
Add conditions to qb for the given item operation scope scope.
static DataStore * self()
Per thread singleton.
Definition: datastore.cpp:215
@ InnerJoin
NOTE: only supported for UPDATE and SELECT queries.
Definition: querybuilder.h:48
void append(const T &value)
void push_back(const T &value)
QHash::iterator find(const Key &key)
Represents a collection of PIM items.
Definition: collection.h:61
PartType fromFqName(const QString &fqName)
Retrieve (or create) PartType for the given fully qualified name.
bool isValid() const const
QDateTime currentDateTimeUtc()
QByteArray toLatin1() const const
An Akonadi Relation.
Definition: relation.h:39
void setColumnValue(const QString &column, const QVariant &value)
Sets a column to the given value (only valid for INSERT and UPDATE queries).
@ LeftJoin
NOTE: only supported for SELECT queries.
Definition: querybuilder.h:50
QHash::iterator insert(const Key &key, const T &value)
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
T value(int i) const const
int size() const const
QDBusConnection sessionBus()
char * toString(const T &value)
QVector< T > result()
Returns the result of this SELECT query.
void finish()
void addValueCondition(const QString &column, CompareOperator op, const QVariant &value)
Add a WHERE condition which compares a column with a given value.
Definition: query.cpp:12
bool contains(const T &value) const const
bool next()
KGuiItem properties()
void reserve(int size)
QString remoteId() const
Returns the remote id of the collection.
Definition: collection.cpp:106
bool exec()
Executes the query, returns true on success.
bool isEmpty() const const
Helper class for creating and executing database SELECT queries.
void addGroupColumns(const QStringList &columns)
Add list of columns to GROUP BY.
QString fromLatin1(const char *str, int size)
QString name(StandardShortcut id)
SortOrder
Sort orders.
Definition: query.h:53
Represents a WHERE condition tree.
Definition: query.h:61
Helper class for retrieving missing items parts from remote resources.
Definition: itemretriever.h:38
void addCondition(const Query::Condition &condition, ConditionType type=WhereCondition)
Add a WHERE condition.
void setSubQueryMode(LogicOperator op)
Set how sub-conditions should be combined, default is And.
Definition: query.cpp:49
An Connection represents one connection of a client to the server.
Definition: connection.h:46
qint64 Id
Describes the unique id type.
Definition: collection.h:79
QString fullName(const PartType &type)
Returns full part name.
T value(int i) const const
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:31
QHash::iterator end()
Helper integration between Akonadi and Qt.
This file is part of the KDE documentation.
Documentation copyright © 1996-2022 The KDE developers.
Generated on Thu Jun 30 2022 03:51:46 by doxygen 1.8.17 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.