Akonadi

itemfetchhelper.cpp
1
2/***************************************************************************
3 * SPDX-FileCopyrightText: 2006-2009 Tobias Koenig <tokoe@kde.org> *
4 * *
5 * SPDX-License-Identifier: LGPL-2.0-or-later *
6 ***************************************************************************/
7
8#include "itemfetchhelper.h"
9
10#include "akonadi.h"
11#include "connection.h"
12#include "handler.h"
13#include "handlerhelper.h"
14#include "shared/akranges.h"
15#include "storage/itemqueryhelper.h"
16#include "storage/itemretrievalmanager.h"
17#include "storage/parttypehelper.h"
18#include "storage/selectquerybuilder.h"
19#include "storage/transaction.h"
20
21#include "agentmanagerinterface.h"
22#include "akonadiserver_debug.h"
23#include "intervalcheck.h"
24#include "utils.h"
25
26#include "private/dbus_p.h"
27
28#include <QDateTime>
29#include <QSqlQuery>
30#include <QStringList>
31#include <QVariant>
32
33#include <QElapsedTimer>
34
35using namespace Akonadi;
36using namespace Akonadi::Server;
37using namespace AkRanges;
38
39#define ENABLE_FETCH_PROFILING 0
40#if ENABLE_FETCH_PROFILING
41#define BEGIN_TIMER(name) \
42 QElapsedTimer name##Timer; \
43 name##Timer.start();
44
45#define END_TIMER(name) const double name##Elapsed = name##Timer.nsecsElapsed() / 1000000.0;
46#define PROF_INC(name) ++name;
47#else
48#define BEGIN_TIMER(name)
49#define END_TIMER(name)
50#define PROF_INC(name)
51#endif
52
53ItemFetchHelper::ItemFetchHelper(Connection *connection,
54 const Scope &scope,
55 const Protocol::ItemFetchScope &itemFetchScope,
56 const Protocol::TagFetchScope &tagFetchScope,
57 AkonadiServer &akonadi,
58 const Protocol::FetchLimit &itemsLimit)
59 : ItemFetchHelper(connection, connection->context(), scope, itemFetchScope, tagFetchScope, akonadi, itemsLimit)
60{
61}
62
63ItemFetchHelper::ItemFetchHelper(Connection *connection,
64 const CommandContext &context,
65 const Scope &scope,
66 const Protocol::ItemFetchScope &itemFetchScope,
67 const Protocol::TagFetchScope &tagFetchScope,
68 AkonadiServer &akonadi,
69 const Protocol::FetchLimit &itemsLimit)
70 : mConnection(connection)
71 , mContext(context)
72 , mScope(scope)
73 , mItemFetchScope(itemFetchScope)
74 , mTagFetchScope(tagFetchScope)
75 , mAkonadi(akonadi)
76 , mItemsLimit(itemsLimit)
77 , mItemQuery(PimItem::tableName())
78 , mPimItemQueryAlias(QLatin1StringView("pimItem_alias"))
79{
80 std::fill(mItemQueryColumnMap, mItemQueryColumnMap + ItemQueryColumnCount, -1);
81}
82
83void ItemFetchHelper::disableATimeUpdates()
84{
85 mUpdateATimeEnabled = false;
86}
87
88enum PartQueryColumns {
89 PartQueryPimIdColumn,
90 PartQueryTypeIdColumn,
91 PartQueryDataColumn,
92 PartQueryStorageColumn,
93 PartQueryVersionColumn,
94 PartQueryDataSizeColumn
95};
96
97QSqlQuery ItemFetchHelper::buildPartQuery(const QList<QByteArray> &partList, bool allPayload, bool allAttrs)
98{
99 /// TODO: merge with ItemQuery
100 QueryBuilder partQuery(PimItem::tableName());
101 if (mItemsLimit.limit() > 0) {
102 partQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
103 }
104
105 if (!partList.isEmpty() || allPayload || allAttrs) {
106 partQuery.addJoin(QueryBuilder::InnerJoin, Part::tableName(), partQuery.getTableWithColumn(PimItem::idColumn()), Part::pimItemIdFullColumnName());
107 partQuery.addColumn(partQuery.getTableWithColumn(PimItem::idColumn()));
108 partQuery.addColumn(Part::partTypeIdFullColumnName());
109 partQuery.addColumn(Part::dataFullColumnName());
110 partQuery.addColumn(Part::storageFullColumnName());
111 partQuery.addColumn(Part::versionFullColumnName());
112 partQuery.addColumn(Part::datasizeFullColumnName());
113
114 partQuery.addSortColumn(partQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
115
116 if (!partList.isEmpty() || allPayload || allAttrs) {
117 Query::Condition cond(Query::Or);
118 for (const QByteArray &b : std::as_const(partList)) {
119 if (b.startsWith("PLD") || b.startsWith("ATR")) {
120 cond.addValueCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartTypeHelper::fromFqName(b).id());
121 }
122 }
123 if (allPayload || allAttrs) {
124 partQuery.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName());
125 if (allPayload) {
126 cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
127 }
128 if (allAttrs) {
129 cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("ATR"));
130 }
131 }
132
133 partQuery.addCondition(cond);
134 }
135
136 ItemQueryHelper::scopeToQuery(mScope, mContext, partQuery);
137
138 if (!partQuery.exec()) {
139 throw HandlerException("Unable to list item parts");
140 }
141 partQuery.query().next();
142 }
143
144 return partQuery.query();
145}
146
147QSqlQuery ItemFetchHelper::buildItemQuery()
148{
149 int column = 0;
150#define ADD_COLUMN(colName, colId) \
151 { \
152 mItemQuery.addColumn(colName); \
153 mItemQueryColumnMap[colId] = column++; \
154 }
155 ADD_COLUMN(PimItem::idFullColumnName(), ItemQueryPimItemIdColumn);
156 if (mItemFetchScope.fetchRemoteId()) {
157 ADD_COLUMN(PimItem::remoteIdFullColumnName(), ItemQueryPimItemRidColumn)
158 }
159 ADD_COLUMN(PimItem::mimeTypeIdFullColumnName(), ItemQueryMimeTypeIdColumn)
160 ADD_COLUMN(PimItem::revFullColumnName(), ItemQueryRevColumn)
161 if (mItemFetchScope.fetchRemoteRevision()) {
162 ADD_COLUMN(PimItem::remoteRevisionFullColumnName(), ItemQueryRemoteRevisionColumn)
163 }
164 if (mItemFetchScope.fetchSize()) {
165 ADD_COLUMN(PimItem::sizeFullColumnName(), ItemQuerySizeColumn)
166 }
167 if (mItemFetchScope.fetchMTime()) {
168 ADD_COLUMN(PimItem::datetimeFullColumnName(), ItemQueryDatetimeColumn)
169 }
170 ADD_COLUMN(PimItem::collectionIdFullColumnName(), ItemQueryCollectionIdColumn)
171 if (mItemFetchScope.fetchGID()) {
172 ADD_COLUMN(PimItem::gidFullColumnName(), ItemQueryPimItemGidColumn)
173 }
174#undef ADD_COLUMN
175
176 mItemQuery.addSortColumn(PimItem::idFullColumnName(), static_cast<Query::SortOrder>(mItemsLimit.sortOrder()));
177 if (mItemsLimit.limit() > 0) {
178 mItemQuery.setLimit(mItemsLimit.limit(), mItemsLimit.limitOffset());
179 }
180
181 ItemQueryHelper::scopeToQuery(mScope, mContext, mItemQuery);
182
183 if (mItemFetchScope.changedSince().isValid()) {
184 mItemQuery.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual, mItemFetchScope.changedSince().toUTC());
185 }
186
187 if (!mItemQuery.exec()) {
188 throw HandlerException("Unable to list items");
189 }
190
191 mItemQuery.query().next();
192
193 return mItemQuery.query();
194}
195
196enum FlagQueryColumns {
197 FlagQueryPimItemIdColumn,
198 FlagQueryFlagIdColumn,
199};
200
201QSqlQuery ItemFetchHelper::buildFlagQuery()
202{
203 QueryBuilder flagQuery(PimItem::tableName());
204 if (mItemsLimit.limit() > 0) {
205 flagQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
206 }
207
208 flagQuery.addJoin(QueryBuilder::InnerJoin,
209 PimItemFlagRelation::tableName(),
210 flagQuery.getTableWithColumn(PimItem::idColumn()),
211 PimItemFlagRelation::leftFullColumnName());
212
213 flagQuery.addColumn(flagQuery.getTableWithColumn(PimItem::idColumn()));
214 flagQuery.addColumn(PimItemFlagRelation::rightFullColumnName());
215
216 ItemQueryHelper::scopeToQuery(mScope, mContext, flagQuery);
217 flagQuery.addSortColumn(flagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
218
219 if (!flagQuery.exec()) {
220 throw HandlerException("Unable to retrieve item flags");
221 }
222
223 flagQuery.query().next();
224
225 return flagQuery.query();
226}
227
228enum TagQueryColumns {
229 TagQueryItemIdColumn,
230 TagQueryTagIdColumn,
231};
232
233QSqlQuery ItemFetchHelper::buildTagQuery()
234{
235 QueryBuilder tagQuery(PimItem::tableName());
236 if (mItemsLimit.limit() > 0) {
237 tagQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
238 }
239
240 tagQuery.addJoin(QueryBuilder::InnerJoin,
241 PimItemTagRelation::tableName(),
242 tagQuery.getTableWithColumn(PimItem::idColumn()),
243 PimItemTagRelation::leftFullColumnName());
244 tagQuery.addJoin(QueryBuilder::InnerJoin, Tag::tableName(), Tag::idFullColumnName(), PimItemTagRelation::rightFullColumnName());
245 tagQuery.addColumn(tagQuery.getTableWithColumn(PimItem::idColumn()));
246 tagQuery.addColumn(Tag::idFullColumnName());
247
248 ItemQueryHelper::scopeToQuery(mScope, mContext, tagQuery);
249 tagQuery.addSortColumn(tagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
250
251 if (!tagQuery.exec()) {
252 throw HandlerException("Unable to retrieve item tags");
253 }
254
255 tagQuery.query().next();
256
257 return tagQuery.query();
258}
259
260enum VRefQueryColumns {
261 VRefQueryCollectionIdColumn,
262 VRefQueryItemIdColumn,
263};
264
265QSqlQuery ItemFetchHelper::buildVRefQuery()
266{
267 QueryBuilder vRefQuery(PimItem::tableName());
268 if (mItemsLimit.limit() > 0) {
269 vRefQuery = QueryBuilder(mItemQuery.query(), mPimItemQueryAlias);
270 }
271
272 vRefQuery.addJoin(QueryBuilder::LeftJoin,
273 CollectionPimItemRelation::tableName(),
274 CollectionPimItemRelation::rightFullColumnName(),
275 vRefQuery.getTableWithColumn(PimItem::idColumn()));
276 vRefQuery.addColumn(CollectionPimItemRelation::leftFullColumnName());
277 vRefQuery.addColumn(CollectionPimItemRelation::rightFullColumnName());
278 ItemQueryHelper::scopeToQuery(mScope, mContext, vRefQuery);
279 vRefQuery.addSortColumn(vRefQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
280
281 if (!vRefQuery.exec()) {
282 throw HandlerException("Unable to retrieve virtual references");
283 }
284
285 vRefQuery.query().next();
286
287 return vRefQuery.query();
288}
289
290bool ItemFetchHelper::isScopeLocal(const Scope &scope)
291{
292 // The only agent allowed to override local scope is the Baloo Indexer
293 if (!mConnection->sessionId().startsWith("akonadi_indexing_agent")) {
294 return false;
295 }
296
297 // Get list of all resources that own all items in the scope
298 QueryBuilder qb(PimItem::tableName(), QueryBuilder::Select);
299 qb.setDistinct(true);
300 qb.addColumn(Resource::nameFullColumnName());
301 qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
302 qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
303 ItemQueryHelper::scopeToQuery(scope, mContext, qb);
304 if (mContext.resource().isValid()) {
305 qb.addValueCondition(Resource::nameFullColumnName(), Query::NotEquals, mContext.resource().name());
306 }
307
308 if (!qb.exec()) {
309 throw HandlerException("Failed to query database");
310 return false;
311 }
312
313 // If there is more than one resource, i.e. this is a fetch from multiple
314 // collections, then don't bother and just return FALSE. This case is aimed
315 // specifically on Baloo, which fetches items from each collection independently,
316 // so it will pass this check.
317 QSqlQuery query = qb.query();
318 if (query.size() != 1) {
319 return false;
320 }
321
322 query.next();
323 const QString resourceName = query.value(0).toString();
324 query.finish();
325
326 org::freedesktop::Akonadi::AgentManager manager(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus());
327 const QString typeIdentifier = manager.agentInstanceType(resourceName);
328 const QVariantMap properties = manager.agentCustomProperties(typeIdentifier);
329 return properties.value(QStringLiteral("HasLocalStorage"), false).toBool();
330}
331
332DataStore *ItemFetchHelper::storageBackend() const
333{
334 if (mConnection) {
335 if (auto store = mConnection->storageBackend()) {
336 return store;
337 }
338 }
339
340 return DataStore::self();
341}
342
343bool ItemFetchHelper::fetchItems(std::function<void(Protocol::FetchItemsResponse &&)> &&itemCallback)
344{
345 BEGIN_TIMER(fetch)
346
347 // retrieve missing parts
348 // HACK: isScopeLocal() is a workaround for resources that have cache expiration
349 // because when the cache expires, Baloo is not able to content of the items. So
350 // we allow fetch of items that belong to local resources (like maildir) to ignore
351 // cacheOnly and retrieve missing parts from the resource. However ItemRetriever
352 // is painfully slow with many items and is generally designed to fetch a few
353 // messages, not all of them. In the long term, we need a better way to do this.
354 BEGIN_TIMER(itemRetriever)
355 BEGIN_TIMER(scopeLocal)
356#if ENABLE_FETCH_PROFILING
357 double scopeLocalElapsed = 0;
358#endif
359 if (!mItemFetchScope.cacheOnly() || isScopeLocal(mScope)) {
360#if ENABLE_FETCH_PROFILING
361 scopeLocalElapsed = scopeLocalTimer.elapsed();
362#endif
363
364 // trigger a collection sync if configured to do so
365 triggerOnDemandFetch();
366
367 // Prepare for a call to ItemRetriever::exec();
368 // From a resource perspective the only parts that can be fetched are payloads.
369 ItemRetriever retriever(mAkonadi.itemRetrievalManager(), mConnection, mContext);
370 retriever.setScope(mScope);
371 retriever.setRetrieveParts(mItemFetchScope.requestedPayloads());
372 retriever.setRetrieveFullPayload(mItemFetchScope.fullPayload());
373 retriever.setChangedSince(mItemFetchScope.changedSince());
374 if (!retriever.exec() && !mItemFetchScope.ignoreErrors()) { // There we go, retrieve the missing parts from the resource.
375 if (mContext.resource().isValid()) {
376 throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1, resource %2) : %3")
377 .arg(mContext.collectionId())
378 .arg(mContext.resource().id())
379 .arg(QString::fromLatin1(retriever.lastError())));
380 } else {
381 throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1) : %2")
382 .arg(mContext.collectionId())
383 .arg(QString::fromLatin1(retriever.lastError())));
384 }
385 }
386 }
387 END_TIMER(itemRetriever)
388
389 BEGIN_TIMER(items)
390 QSqlQuery itemQuery = buildItemQuery();
391 END_TIMER(items)
392
393 // error if query did not find any item and scope is not listing items but
394 // a request for a specific item
395 if (!itemQuery.isValid()) {
396 if (mItemFetchScope.ignoreErrors()) {
397 return true;
398 }
399 switch (mScope.scope()) {
400 case Scope::Uid: // fall through
401 case Scope::Rid: // fall through
402 case Scope::HierarchicalRid: // fall through
403 case Scope::Gid:
404 throw HandlerException("Item query returned empty result set");
405 break;
406 default:
407 break;
408 }
409 }
410 // build part query if needed
411 BEGIN_TIMER(parts)
412 QSqlQuery partQuery(storageBackend()->database());
413 if (!mItemFetchScope.requestedParts().isEmpty() || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
414 partQuery = buildPartQuery(mItemFetchScope.requestedParts(), mItemFetchScope.fullPayload(), mItemFetchScope.allAttributes());
415 }
416 END_TIMER(parts)
417
418 // build flag query if needed
419 BEGIN_TIMER(flags)
420 QSqlQuery flagQuery(storageBackend()->database());
421 if (mItemFetchScope.fetchFlags()) {
422 flagQuery = buildFlagQuery();
423 }
424 END_TIMER(flags)
425
426 // build tag query if needed
427 BEGIN_TIMER(tags)
428 QSqlQuery tagQuery(storageBackend()->database());
429 if (mItemFetchScope.fetchTags()) {
430 tagQuery = buildTagQuery();
431 }
432 END_TIMER(tags)
433
434 BEGIN_TIMER(vRefs)
435 QSqlQuery vRefQuery(storageBackend()->database());
436 if (mItemFetchScope.fetchVirtualReferences()) {
437 vRefQuery = buildVRefQuery();
438 }
439 END_TIMER(vRefs)
440
441#if ENABLE_FETCH_PROFILING
442 int itemsCount = 0;
443 int flagsCount = 0;
444 int partsCount = 0;
445 int tagsCount = 0;
446 int vRefsCount = 0;
447#endif
448
449 BEGIN_TIMER(processing)
450 QHash<qint64, QByteArray> flagIdNameCache;
451 QHash<qint64, QString> mimeTypeIdNameCache;
452 QHash<qint64, QByteArray> partTypeIdNameCache;
453 while (itemQuery.isValid()) {
454 PROF_INC(itemsCount)
455
456 const qint64 pimItemId = extractQueryResult(itemQuery, ItemQueryPimItemIdColumn).toLongLong();
457 const int pimItemRev = extractQueryResult(itemQuery, ItemQueryRevColumn).toInt();
458
459 Protocol::FetchItemsResponse response;
460 response.setId(pimItemId);
461 response.setRevision(pimItemRev);
462 const qint64 mimeTypeId = extractQueryResult(itemQuery, ItemQueryMimeTypeIdColumn).toLongLong();
463 auto mtIter = mimeTypeIdNameCache.find(mimeTypeId);
464 if (mtIter == mimeTypeIdNameCache.end()) {
465 mtIter = mimeTypeIdNameCache.insert(mimeTypeId, MimeType::retrieveById(mimeTypeId).name());
466 }
467 response.setMimeType(mtIter.value());
468 if (mItemFetchScope.fetchRemoteId()) {
469 response.setRemoteId(extractQueryResult(itemQuery, ItemQueryPimItemRidColumn).toString());
470 }
471 response.setParentId(extractQueryResult(itemQuery, ItemQueryCollectionIdColumn).toLongLong());
472
473 if (mItemFetchScope.fetchSize()) {
474 response.setSize(extractQueryResult(itemQuery, ItemQuerySizeColumn).toLongLong());
475 }
476 if (mItemFetchScope.fetchMTime()) {
477 response.setMTime(extractQueryResult(itemQuery, ItemQueryDatetimeColumn).toDateTime());
478 }
479 if (mItemFetchScope.fetchRemoteRevision()) {
480 response.setRemoteRevision(extractQueryResult(itemQuery, ItemQueryRemoteRevisionColumn).toString());
481 }
482 if (mItemFetchScope.fetchGID()) {
483 response.setGid(extractQueryResult(itemQuery, ItemQueryPimItemGidColumn).toString());
484 }
485
486 if (mItemFetchScope.fetchFlags()) {
487 QList<QByteArray> flags;
488 while (flagQuery.isValid()) {
489 const qint64 id = flagQuery.value(FlagQueryPimItemIdColumn).toLongLong();
490 if (id > pimItemId) {
491 flagQuery.next();
492 continue;
493 } else if (id < pimItemId) {
494 break;
495 }
496 const qint64 flagId = flagQuery.value(FlagQueryFlagIdColumn).toLongLong();
497 auto flagNameIter = flagIdNameCache.find(flagId);
498 if (flagNameIter == flagIdNameCache.end()) {
499 flagNameIter = flagIdNameCache.insert(flagId, Flag::retrieveById(flagId).name().toUtf8());
500 }
501 flags << flagNameIter.value();
502 flagQuery.next();
503 }
504 response.setFlags(flags);
505 }
506
507 if (mItemFetchScope.fetchTags()) {
508 QList<qint64> tagIds;
509 QList<Protocol::FetchTagsResponse> tags;
510 while (tagQuery.isValid()) {
511 PROF_INC(tagsCount)
512 const qint64 id = tagQuery.value(TagQueryItemIdColumn).toLongLong();
513 if (id > pimItemId) {
514 tagQuery.next();
515 continue;
516 } else if (id < pimItemId) {
517 break;
518 }
519 tagIds << tagQuery.value(TagQueryTagIdColumn).toLongLong();
520 tagQuery.next();
521 }
522
523 if (mTagFetchScope.fetchIdOnly()) {
524 tags = tagIds | Views::transform([](const auto tagId) {
525 Protocol::FetchTagsResponse resp;
526 resp.setId(tagId);
527 return resp;
528 })
529 | Actions::toQVector;
530 } else {
531 tags = tagIds | Views::transform([this](const auto tagId) {
532 return HandlerHelper::fetchTagsResponse(Tag::retrieveById(tagId), mTagFetchScope, mConnection);
533 })
534 | Actions::toQVector;
535 }
536 response.setTags(tags);
537 }
538
539 if (mItemFetchScope.fetchVirtualReferences()) {
540 QList<qint64> vRefs;
541 while (vRefQuery.isValid()) {
542 PROF_INC(vRefsCount)
543 const qint64 id = vRefQuery.value(VRefQueryItemIdColumn).toLongLong();
544 if (id > pimItemId) {
545 vRefQuery.next();
546 continue;
547 } else if (id < pimItemId) {
548 break;
549 }
550 vRefs << vRefQuery.value(VRefQueryCollectionIdColumn).toLongLong();
551 vRefQuery.next();
552 }
553 response.setVirtualReferences(vRefs);
554 }
555
556 if (mItemFetchScope.fetchRelations()) {
557 SelectQueryBuilder<Relation> qb;
558 Query::Condition condition;
559 condition.setSubQueryMode(Query::Or);
560 condition.addValueCondition(Relation::leftIdFullColumnName(), Query::Equals, pimItemId);
561 condition.addValueCondition(Relation::rightIdFullColumnName(), Query::Equals, pimItemId);
562 qb.addCondition(condition);
563 qb.addGroupColumns(QStringList() << Relation::leftIdColumn() << Relation::rightIdColumn() << Relation::typeIdColumn()
564 << Relation::remoteIdColumn());
565 if (!qb.exec()) {
566 throw HandlerException("Unable to list item relations");
567 }
568 QList<Protocol::FetchRelationsResponse> relations;
569 const auto result = qb.result();
570 relations.reserve(result.size());
571 for (const Relation &rel : result) {
572 relations.push_back(HandlerHelper::fetchRelationsResponse(rel));
573 ;
574 }
575 response.setRelations(relations);
576 }
577
578 if (mItemFetchScope.ancestorDepth() != Protocol::ItemFetchScope::NoAncestor) {
579 response.setAncestors(ancestorsForItem(response.parentId()));
580 }
581
582 bool skipItem = false;
583
584 QList<QByteArray> cachedParts;
585 QList<Protocol::StreamPayloadResponse> parts;
586 while (partQuery.isValid()) {
587 PROF_INC(partsCount)
588 const qint64 id = partQuery.value(PartQueryPimIdColumn).toLongLong();
589 if (id > pimItemId) {
590 partQuery.next();
591 continue;
592 } else if (id < pimItemId) {
593 break;
594 }
595
596 const qint64 partTypeId = partQuery.value(PartQueryTypeIdColumn).toLongLong();
597 auto ptIter = partTypeIdNameCache.find(partTypeId);
598 if (ptIter == partTypeIdNameCache.end()) {
599 ptIter = partTypeIdNameCache.insert(partTypeId, PartTypeHelper::fullName(PartType::retrieveById(partTypeId)).toUtf8());
600 }
601 Protocol::PartMetaData metaPart;
602 Protocol::StreamPayloadResponse partData;
603 partData.setPayloadName(ptIter.value());
604 metaPart.setName(ptIter.value());
605 metaPart.setVersion(partQuery.value(PartQueryVersionColumn).toInt());
606 metaPart.setSize(partQuery.value(PartQueryDataSizeColumn).toLongLong());
607
608 const QByteArray data = Utils::variantToByteArray(partQuery.value(PartQueryDataColumn));
609 if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
610 if (!data.isEmpty()) {
611 cachedParts << ptIter.value();
612 }
613 partQuery.next();
614 } else {
615 if (mItemFetchScope.ignoreErrors() && data.isEmpty()) {
616 // We wanted the payload, couldn't get it, and are ignoring errors. Skip the item.
617 // This is not an error though, it's fine to have empty payload parts (to denote existing but not cached parts)
618 qCDebug(AKONADISERVER_LOG) << "item" << id << "has an empty payload part in parttable for part" << metaPart.name();
619 skipItem = true;
620 break;
621 }
622 metaPart.setStorageType(static_cast<Protocol::PartMetaData::StorageType>(partQuery.value(PartQueryStorageColumn).toInt()));
623 if (data.isEmpty()) {
624 partData.setData(QByteArray(""));
625 } else {
626 partData.setData(data);
627 }
628 partData.setMetaData(metaPart);
629
630 if (mItemFetchScope.requestedParts().contains(ptIter.value()) || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
631 parts.append(partData);
632 }
633
634 partQuery.next();
635 }
636 }
637 response.setParts(parts);
638
639 if (skipItem) {
640 itemQuery.next();
641 continue;
642 }
643
644 if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
645 response.setCachedParts(cachedParts);
646 }
647
648 if (itemCallback) {
649 itemCallback(std::move(response));
650 } else {
651 mConnection->sendResponse(std::move(response));
652 }
653
654 itemQuery.next();
655 }
656 tagQuery.finish();
657 flagQuery.finish();
658 partQuery.finish();
659 vRefQuery.finish();
660 itemQuery.finish();
661 END_TIMER(processing)
662
663 // update atime (only if the payload was actually requested, otherwise a simple resource sync prevents cache clearing)
664 BEGIN_TIMER(aTime)
665 if (mUpdateATimeEnabled && (needsAccessTimeUpdate(mItemFetchScope.requestedParts()) || mItemFetchScope.fullPayload())) {
666 updateItemAccessTime();
667 }
668 END_TIMER(aTime)
669
670 END_TIMER(fetch)
671#if ENABLE_FETCH_PROFILING
672 qCDebug(AKONADISERVER_LOG) << "ItemFetchHelper execution stats:";
673 qCDebug(AKONADISERVER_LOG) << "\tItems query:" << itemsElapsed << "ms," << itemsCount << " items in total";
674 qCDebug(AKONADISERVER_LOG) << "\tFlags query:" << flagsElapsed << "ms, " << flagsCount << " flags in total";
675 qCDebug(AKONADISERVER_LOG) << "\tParts query:" << partsElapsed << "ms, " << partsCount << " parts in total";
676 qCDebug(AKONADISERVER_LOG) << "\tTags query: " << tagsElapsed << "ms, " << tagsCount << " tags in total";
677 qCDebug(AKONADISERVER_LOG) << "\tVRefs query:" << vRefsElapsed << "ms, " << vRefsCount << " vRefs in total";
678 qCDebug(AKONADISERVER_LOG) << "\t------------";
679 qCDebug(AKONADISERVER_LOG) << "\tItem retriever:" << itemRetrieverElapsed << "ms (scope local:" << scopeLocalElapsed << "ms)";
680 qCDebug(AKONADISERVER_LOG) << "\tTotal query:" << (itemsElapsed + flagsElapsed + partsElapsed + tagsElapsed + vRefsElapsed) << "ms";
681 qCDebug(AKONADISERVER_LOG) << "\tTotal processing: " << processingElapsed << "ms";
682 qCDebug(AKONADISERVER_LOG) << "\tATime update:" << aTimeElapsed << "ms";
683 qCDebug(AKONADISERVER_LOG) << "\t============";
684 qCDebug(AKONADISERVER_LOG) << "\tTotal FETCH:" << fetchElapsed << "ms";
685 qCDebug(AKONADISERVER_LOG);
686 qCDebug(AKONADISERVER_LOG);
687#endif
688
689 return true;
690}
691
692bool ItemFetchHelper::needsAccessTimeUpdate(const QList<QByteArray> &parts)
693{
694 // TODO technically we should compare the part list with the cache policy of
695 // the parent collection of the retrieved items, but that's kinda expensive
696 // Only updating the atime if the full payload was requested is a good
697 // approximation though.
698 return parts.contains(AKONADI_PARAM_PLD_RFC822);
699}
700
701void ItemFetchHelper::updateItemAccessTime()
702{
703 Transaction transaction(storageBackend(), QStringLiteral("update atime"));
704 QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
705 qb.setColumnValue(PimItem::atimeColumn(), QDateTime::currentDateTimeUtc());
706 ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
707
708 if (!qb.exec()) {
709 qCWarning(AKONADISERVER_LOG) << "Unable to update item access time";
710 } else {
711 transaction.commit();
712 }
713}
714
715void ItemFetchHelper::triggerOnDemandFetch()
716{
717 if (mContext.collectionId() <= 0 || mItemFetchScope.cacheOnly()) {
718 return;
719 }
720
721 Collection collection = mContext.collection();
722
723 // HACK: don't trigger on-demand syncing if the resource is the one triggering it
724 if (mConnection->sessionId() == collection.resource().name().toLatin1()) {
725 return;
726 }
727
728 storageBackend()->activeCachePolicy(collection);
729 if (!collection.cachePolicySyncOnDemand()) {
730 return;
731 }
732
733 mConnection->akonadi().intervalChecker().requestCollectionSync(collection);
734}
735
736QList<Protocol::Ancestor> ItemFetchHelper::ancestorsForItem(Collection::Id parentColId)
737{
738 if (mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::NoAncestor || parentColId == 0) {
739 return QList<Protocol::Ancestor>();
740 }
741 const auto it = mAncestorCache.constFind(parentColId);
742 if (it != mAncestorCache.cend()) {
743 return *it;
744 }
745
746 QList<Protocol::Ancestor> ancestors;
747 Collection col = Collection::retrieveById(parentColId);
748 const int depthNum = mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::ParentAncestor ? 1 : INT_MAX;
749 for (int i = 0; i < depthNum; ++i) {
750 if (!col.isValid()) {
751 Protocol::Ancestor ancestor;
752 ancestor.setId(0);
753 ancestors << ancestor;
754 break;
755 }
756 Protocol::Ancestor ancestor;
757 ancestor.setId(col.id());
758 ancestor.setRemoteId(col.remoteId());
759 ancestors << ancestor;
760 col = col.parent();
761 }
762 mAncestorCache.insert(parentColId, ancestors);
763 return ancestors;
764}
765
766QVariant ItemFetchHelper::extractQueryResult(const QSqlQuery &query, ItemFetchHelper::ItemQueryColumns column) const
767{
768 const int colId = mItemQueryColumnMap[column];
769 Q_ASSERT(colId >= 0);
770 return query.value(colId);
771}
Akonadi::Collection
Represents a collection of PIM items.
Definition collection.h:62
Akonadi::Collection::Id
qint64 Id
Describes the unique id type.
Definition collection.h:79
Akonadi::Collection::remoteId
QString remoteId() const
Returns the remote id of the collection.
Definition collection.cpp:106
Akonadi::Relation
An Akonadi Relation.
Definition relation.h:41
Akonadi::Server::Connection
An Connection represents one connection of a client to the server.
Definition connection.h:39
Akonadi::Server::DataStore
This class handles all the database access.
Definition datastore.h:95
Akonadi::Server::DataStore::self
static DataStore * self()
Per thread singleton.
Definition datastore.cpp:248
Akonadi::Server::DataStore::activeCachePolicy
virtual void activeCachePolicy(Collection &col)
Determines the active cache policy for this Collection.
Definition datastore.cpp:980
Akonadi::Server::IntervalCheck::requestCollectionSync
void requestCollectionSync(const Collection &collection)
Requests the given collection to be synced.
Definition intervalcheck.cpp:29
Akonadi::Server::ItemRetriever
Helper class for retrieving missing items parts from remote resources.
Definition itemretriever.h:38
Akonadi::Server::QueryBuilder
Helper class to construct arbitrary SQL queries.
Definition querybuilder.h:33
Akonadi::Server::QueryBuilder::addValueCondition
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
Definition querybuilder.cpp:182
Akonadi::Server::QueryBuilder::addSortColumn
void addSortColumn(const QString &column, Query::SortOrder order=Query::Ascending)
Add sort column.
Definition querybuilder.cpp:638
Akonadi::Server::QueryBuilder::exec
bool exec()
Executes the query, returns true on success.
Definition querybuilder.cpp:396
Akonadi::Server::QueryBuilder::addCondition
void addCondition(const Query::Condition &condition, ConditionType type=WhereCondition)
Add a WHERE condition.
Definition querybuilder.cpp:632
Akonadi::Server::QueryBuilder::setColumnValue
void setColumnValue(const QString &column, const QVariant &value)
Sets a column to the given value (only valid for INSERT and UPDATE queries).
Definition querybuilder.cpp:655
Akonadi::Server::QueryBuilder::addGroupColumns
void addGroupColumns(const QStringList &columns)
Add list of columns to GROUP BY.
Definition querybuilder.cpp:649
Akonadi::Server::QueryBuilder::query
QSqlQuery & query()
Returns the query, only valid after exec().
Definition querybuilder.cpp:200
Akonadi::Server::QueryBuilder::InnerJoin
@ InnerJoin
NOTE: only supported for UPDATE and SELECT queries.
Definition querybuilder.h:49
Akonadi::Server::QueryBuilder::LeftJoin
@ LeftJoin
NOTE: only supported for SELECT queries.
Definition querybuilder.h:51
Akonadi::Server::QueryBuilder::setLimit
void setLimit(int limit, int offset=-1)
Limits the amount of retrieved rows.
Definition querybuilder.cpp:665
Akonadi::Server::Query::Condition
Represents a WHERE condition tree.
Definition query.h:62
Akonadi::Server::Query::Condition::setSubQueryMode
void setSubQueryMode(LogicOperator op)
Set how sub-conditions should be combined, default is And.
Definition query.cpp:49
Akonadi::Server::Query::Condition::addValueCondition
void addValueCondition(const QString &column, CompareOperator op, const QVariant &value)
Add a WHERE condition which compares a column with a given value.
Definition query.cpp:12
Akonadi::Server::SelectQueryBuilder
Helper class for creating and executing database SELECT queries.
Definition selectquerybuilder.h:21
Akonadi::Server::SelectQueryBuilder::result
QList< T > result()
Returns the result of this SELECT query.
Definition selectquerybuilder.h:40
Akonadi::Server::Transaction
Helper class for DataStore transaction handling.
Definition transaction.h:23
Resource::name
QString name() const
Akonadi::Server::ItemQueryHelper::scopeToQuery
void scopeToQuery(const Scope &scope, const CommandContext &context, QueryBuilder &qb)
Add conditions to qb for the given item operation scope scope.
Definition itemqueryhelper.cpp:109
Akonadi::Server::PartTypeHelper::fromFqName
PartType fromFqName(const QString &fqName)
Retrieve (or create) PartType for the given fully qualified name.
Definition parttypehelper.cpp:27
Akonadi::Server::PartTypeHelper::fullName
QString fullName(const PartType &type)
Returns full part name.
Definition parttypehelper.cpp:66
Akonadi::Server::Query::SortOrder
SortOrder
Sort orders.
Definition query.h:53
Akonadi
Helper integration between Akonadi and Qt.
Baloo::toString
char * toString(const EngineQuery &query)
KApplicationTrader::query
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
KStandardGuiItem::properties
KGuiItem properties()
KStandardShortcut::name
QString name(StandardShortcut id)
std
A glue between Qt and the standard library.
QByteArray::isEmpty
bool isEmpty() const const
QByteArray::startsWith
bool startsWith(QByteArrayView bv) const const
QDateTime::currentDateTimeUtc
QDateTime currentDateTimeUtc()
QDBusConnection::sessionBus
QDBusConnection sessionBus()
QHash::cend
const_iterator cend() const const
QHash::constFind
const_iterator constFind(const Key &key) const const
QHash::end
iterator end()
QHash::find
iterator find(const Key &key)
QHash::insert
iterator insert(const Key &key, const T &value)
QList::append
void append(QList< T > &&value)
QList::contains
bool contains(const AT &value) const const
QList::isEmpty
bool isEmpty() const const
QList::push_back
void push_back(parameter_type value)
QList::reserve
void reserve(qsizetype size)
QList::size
qsizetype size() const const
QList::value
T value(qsizetype i) const const
QObject::qobject_cast
T qobject_cast(QObject *object)
QSqlQuery::finish
void finish()
QSqlQuery::isValid
bool isValid() const const
QSqlQuery::next
bool next()
QString::fromLatin1
QString fromLatin1(QByteArrayView str)
QString::toLatin1
QByteArray toLatin1() const const
QVariant::toInt
int toInt(bool *ok) const const
QVariant::toLongLong
qlonglong toLongLong(bool *ok) const const
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Tue Mar 26 2024 11:13:38 by doxygen 1.10.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.