Akonadi

collectionfetchhandler.cpp
1/*
2 SPDX-FileCopyrightText: 2007 Volker Krause <vkrause@kde.org>
3
4 SPDX-License-Identifier: LGPL-2.0-or-later
5*/
6
7#include "collectionfetchhandler.h"
8#include "akonadiserver_debug.h"
9
10#include "connection.h"
11#include "handlerhelper.h"
12#include "storage/collectionqueryhelper.h"
13#include "storage/datastore.h"
14#include "storage/selectquerybuilder.h"
15
16#include "private/scope_p.h"
17
18using namespace Akonadi;
19using namespace Akonadi::Server;
20
21template<typename T>
22static bool intersect(const QList<typename T::Id> &l1, const QList<T> &l2)
23{
24 for (const T &e2 : l2) {
25 if (l1.contains(e2.id())) {
26 return true;
27 }
28 }
29 return false;
30}
31
32[[nodiscard]] static bool isRootCollection(const Scope &scope)
33{
34 return scope.isEmpty() || (scope.scope() == Scope::Uid && scope.uidSet().size() == 1 && scope.uid() == 0);
35}
36
37CollectionFetchHandler::CollectionFetchHandler(AkonadiServer &akonadi)
38 : Handler(akonadi)
39{
40}
41
42QStack<Collection> CollectionFetchHandler::ancestorsForCollection(const Collection &col)
43{
44 if (mAncestorDepth <= 0) {
45 return QStack<Collection>();
46 }
47 QStack<Collection> ancestors;
48 Collection parent = col;
49 for (int i = 0; i < mAncestorDepth; ++i) {
50 if (parent.parentId() == 0) {
51 break;
52 }
53 if (mAncestors.contains(parent.parentId())) {
54 parent = mAncestors.value(parent.parentId());
55 } else {
56 parent = mCollections.value(parent.parentId());
57 }
58 if (!parent.isValid()) {
59 qCWarning(AKONADISERVER_LOG) << "Found an invalid parent in ancestors of Collection" << col.name() << "(ID:" << col.id() << ")";
60 throw HandlerException("Found invalid parent in ancestors");
61 }
62 ancestors.prepend(parent);
63 }
64 return ancestors;
65}
66
67CollectionAttribute::List CollectionFetchHandler::getAttributes(const Collection &col, const QSet<QByteArray> &filter)
68{
69 CollectionAttribute::List attributes;
70 auto it = mCollectionAttributes.find(col.id());
71 while (it != mCollectionAttributes.end() && it.key() == col.id()) {
72 if (filter.isEmpty() || filter.contains(it.value().type())) {
73 attributes << it.value();
74 }
75 ++it;
76 }
77
78 {
79 CollectionAttribute attr;
80 attr.setType(AKONADI_PARAM_ENABLED);
81 attr.setValue(col.enabled() ? "TRUE" : "FALSE");
82 attributes << attr;
83 }
84
85 return attributes;
86}
87
88void CollectionFetchHandler::listCollection(const Collection &root,
89 const QStack<Collection> &ancestors,
90 const QStringList &mimeTypes,
91 const CollectionAttribute::List &attributes)
92{
93 QStack<CollectionAttribute::List> ancestorAttributes;
94 // backwards compatibility, collectionToByteArray will automatically fall-back to id + remoteid
95 if (!mAncestorAttributes.isEmpty()) {
96 ancestorAttributes.reserve(ancestors.size());
97 for (const Collection &col : ancestors) {
98 ancestorAttributes.push(getAttributes(col, mAncestorAttributes));
99 }
100 }
101
102 // write out collection details
103 Collection dummy = root;
104 storageBackend()->activeCachePolicy(dummy);
105
106 sendResponse(
107 HandlerHelper::fetchCollectionsResponse(akonadi(), dummy, attributes, mIncludeStatistics, mAncestorDepth, ancestors, ancestorAttributes, mimeTypes));
108}
109
110static Query::Condition filterCondition(const QString &column)
111{
112 Query::Condition orCondition(Query::Or);
113 orCondition.addValueCondition(column, Query::Equals, static_cast<int>(Collection::True));
114 Query::Condition andCondition(Query::And);
115 andCondition.addValueCondition(column, Query::Equals, static_cast<int>(Collection::Undefined));
116 andCondition.addValueCondition(Collection::enabledFullColumnName(), Query::Equals, true);
117 orCondition.addCondition(andCondition);
118 return orCondition;
119}
120
121bool CollectionFetchHandler::checkFilterCondition(const Collection &col) const
122{
123 // Don't include the collection when only looking for enabled collections
124 if (mEnabledCollections && !col.enabled()) {
125 return false;
126 }
127 // Don't include the collection when only looking for collections to display/index/sync
128 if (mCollectionsToDisplay && (((col.displayPref() == Collection::Undefined) && !col.enabled()) || (col.displayPref() == Collection::False))) {
129 return false;
130 }
131 if (mCollectionsToIndex && (((col.indexPref() == Collection::Undefined) && !col.enabled()) || (col.indexPref() == Collection::False))) {
132 return false;
133 }
134 // Single collection sync will still work since that is using a base fetch
135 if (mCollectionsToSynchronize && (((col.syncPref() == Collection::Undefined) && !col.enabled()) || (col.syncPref() == Collection::False))) {
136 return false;
137 }
138 return true;
139}
140
141static QueryBuilder getAttributeQuery(const QVariantList &ids, const QSet<QByteArray> &requestedAttributes)
142{
143 QueryBuilder qb(CollectionAttribute::tableName());
144
145 qb.addValueCondition(CollectionAttribute::collectionIdFullColumnName(), Query::In, ids);
146
147 qb.addColumn(CollectionAttribute::collectionIdFullColumnName());
148 qb.addColumn(CollectionAttribute::typeFullColumnName());
149 qb.addColumn(CollectionAttribute::valueFullColumnName());
150
151 if (!requestedAttributes.isEmpty()) {
152 QVariantList attributes;
153 attributes.reserve(requestedAttributes.size());
154 for (const QByteArray &type : requestedAttributes) {
155 attributes << type;
156 }
157 qb.addValueCondition(CollectionAttribute::typeFullColumnName(), Query::In, attributes);
158 }
159
160 qb.addSortColumn(CollectionAttribute::collectionIdFullColumnName(), Query::Ascending);
161
162 if (!qb.exec()) {
163 throw HandlerException("Unable to retrieve attributes for listing");
164 }
165 return qb;
166}
167
168void CollectionFetchHandler::retrieveAttributes(const QVariantList &collectionIds)
169{
170 // We are querying for the attributes in batches because something can't handle WHERE IN queries with sets larger than 999
171 int start = 0;
172 const int size = 999;
173 while (start < collectionIds.size()) {
174 const QVariantList ids = collectionIds.mid(start, size);
175 auto attributeQb = getAttributeQuery(ids, mAncestorAttributes);
176 auto &attributeQuery = attributeQb.query();
177 while (attributeQuery.next()) {
178 CollectionAttribute attr;
179 attr.setType(attributeQuery.value(1).toByteArray());
180 attr.setValue(attributeQuery.value(2).toByteArray());
181 // qCDebug(AKONADISERVER_LOG) << "found attribute " << attr.type() << attr.value();
182 mCollectionAttributes.insert(attributeQuery.value(0).toLongLong(), attr);
183 }
184 start += size;
185 }
186}
187
188static QueryBuilder getMimeTypeQuery(const QVariantList &ids)
189{
190 QueryBuilder qb(CollectionMimeTypeRelation::tableName());
191
192 qb.addJoin(QueryBuilder::LeftJoin, MimeType::tableName(), MimeType::idFullColumnName(), CollectionMimeTypeRelation::rightFullColumnName());
193 qb.addValueCondition(CollectionMimeTypeRelation::leftFullColumnName(), Query::In, ids);
194
195 qb.addColumn(CollectionMimeTypeRelation::leftFullColumnName());
196 qb.addColumn(CollectionMimeTypeRelation::rightFullColumnName());
197 qb.addColumn(MimeType::nameFullColumnName());
198 qb.addSortColumn(CollectionMimeTypeRelation::leftFullColumnName(), Query::Ascending);
199
200 if (!qb.exec()) {
201 throw HandlerException("Unable to retrieve mimetypes for listing");
202 }
203 return qb;
204}
205
206void CollectionFetchHandler::retrieveCollections(const Collection &topParent, int depth)
207{
208 /*
209 * Retrieval of collections:
210 * The aim is to reduce the amount of queries as much as possible, as this has the largest performance impact for large queries.
211 * * First all collections that match the given criteria are queried
212 * * We then filter the false positives:
213 * ** all collections out that are not part of the tree we asked for are filtered
214 * * Finally we complete the tree by adding missing collections
215 *
216 * Mimetypes and attributes are also retrieved in single queries to avoid spawning two queries per collection (the N+1 problem).
217 * Note that we're not querying attributes and mimetypes for the collections that are only included to complete the tree,
218 * this results in no items being queried for those collections.
219 */
220
221 const qint64 parentId = topParent.isValid() ? topParent.id() : 0;
222 {
224
225 if (depth == 0) {
226 qb.addValueCondition(Collection::idFullColumnName(), Query::Equals, parentId);
227 } else if (depth == 1) {
228 if (topParent.isValid()) {
229 qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Equals, parentId);
230 } else {
231 qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant());
232 }
233 } else {
234 if (topParent.isValid()) {
235 qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, topParent.resourceId());
236 } else {
237 // Gimme gimme gimme...everything!
238 }
239 }
240
241 // Base listings should succeed always
242 if (depth != 0) {
243 if (mCollectionsToSynchronize) {
244 qb.addCondition(filterCondition(Collection::syncPrefFullColumnName()));
245 } else if (mCollectionsToDisplay) {
246 qb.addCondition(filterCondition(Collection::displayPrefFullColumnName()));
247 } else if (mCollectionsToIndex) {
248 qb.addCondition(filterCondition(Collection::indexPrefFullColumnName()));
249 } else if (mEnabledCollections) {
250 qb.addValueCondition(Collection::enabledFullColumnName(), Query::Equals, true);
251 }
252 if (mResource.isValid()) {
253 qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, mResource.id());
254 }
255
256 if (!mMimeTypes.isEmpty()) {
258 CollectionMimeTypeRelation::tableName(),
259 CollectionMimeTypeRelation::leftColumn(),
260 Collection::idFullColumnName());
261 QVariantList mimeTypeFilter;
262 mimeTypeFilter.reserve(mMimeTypes.size());
263 for (MimeType::Id mtId : std::as_const(mMimeTypes)) {
264 mimeTypeFilter << mtId;
265 }
266 qb.addValueCondition(CollectionMimeTypeRelation::rightColumn(), Query::In, mimeTypeFilter);
267 qb.addGroupColumn(Collection::idFullColumnName());
268 }
269 }
270
271 if (!qb.exec()) {
272 throw HandlerException("Unable to retrieve collection for listing");
273 }
274 const auto result{qb.result()};
275 for (const Collection &col : result) {
276 mCollections.insert(col.id(), col);
277 }
278 }
279
280 // Post filtering that we couldn't do as part of the sql query
281 if (depth > 0) {
282 auto it = mCollections.begin();
283 while (it != mCollections.end()) {
284 if (topParent.isValid()) {
285 // Check that each collection is linked to the root collection
286 bool foundParent = false;
287 // We iterate over parents to link it to topParent if possible
288 Collection::Id id = it->parentId();
289 while (id > 0) {
290 if (id == parentId) {
291 foundParent = true;
292 break;
293 }
294 Collection col = mCollections.value(id);
295 if (!col.isValid()) {
296 col = Collection::retrieveById(id);
297 }
298 id = col.parentId();
299 }
300 if (!foundParent) {
301 it = mCollections.erase(it);
302 continue;
303 }
304 }
305 ++it;
306 }
307 }
308
309 QVariantList mimeTypeIds;
310 QVariantList attributeIds;
311 QVariantList ancestorIds;
312 const auto collectionSize{mCollections.size()};
313 mimeTypeIds.reserve(collectionSize);
314 attributeIds.reserve(collectionSize);
315 // We'd only require the non-leaf collections, but we don't know which those are, so we take all.
316 ancestorIds.reserve(collectionSize);
317 for (auto it = mCollections.cbegin(), end = mCollections.cend(); it != end; ++it) {
318 mimeTypeIds << it.key();
319 attributeIds << it.key();
320 ancestorIds << it.key();
321 }
322
323 if (mAncestorDepth > 0 && topParent.isValid()) {
324 // unless depth is 0 the base collection is not part of the listing
325 mAncestors.insert(topParent.id(), topParent);
326 ancestorIds << topParent.id();
327 // We need to retrieve additional ancestors to what we already have in the tree
328 Collection parent = topParent;
329 for (int i = 0; i < mAncestorDepth; ++i) {
330 if (parent.parentId() == 0) {
331 break;
332 }
333 parent = parent.parent();
334 mAncestors.insert(parent.id(), parent);
335 // We also require the attributes
336 ancestorIds << parent.id();
337 }
338 }
339
340 QSet<qint64> missingCollections;
341 if (depth > 0) {
342 for (const Collection &col : std::as_const(mCollections)) {
343 if (col.parentId() != parentId && !mCollections.contains(col.parentId())) {
344 missingCollections.insert(col.parentId());
345 }
346 }
347 }
348
349 /*
350 QSet<qint64> knownIds;
351 for (const Collection &col : mCollections) {
352 knownIds.insert(col.id());
353 }
354 qCDebug(AKONADISERVER_LOG) << "HAS:" << knownIds;
355 qCDebug(AKONADISERVER_LOG) << "MISSING:" << missingCollections;
356 */
357
358 // Fetch missing collections that are part of the tree
359 while (!missingCollections.isEmpty()) {
361 QVariantList ids;
362 ids.reserve(missingCollections.size());
363 for (qint64 id : std::as_const(missingCollections)) {
364 ids << id;
365 }
366 qb.addValueCondition(Collection::idFullColumnName(), Query::In, ids);
367 if (!qb.exec()) {
368 throw HandlerException("Unable to retrieve collections for listing");
369 }
370
371 missingCollections.clear();
372 const auto missingCols = qb.result();
373 for (const Collection &missingCol : missingCols) {
374 mCollections.insert(missingCol.id(), missingCol);
375 ancestorIds << missingCol.id();
376 attributeIds << missingCol.id();
377 mimeTypeIds << missingCol.id();
378 // We have to do another round if the parents parent is missing
379 if (missingCol.parentId() != parentId && !mCollections.contains(missingCol.parentId())) {
380 missingCollections.insert(missingCol.parentId());
381 }
382 }
383 }
384
385 // Since we don't know when we'll need the ancestor attributes, we have to fetch them all together.
386 // The alternative would be to query for each collection which would reintroduce the N+1 query performance problem.
387 if (!mAncestorAttributes.isEmpty()) {
388 retrieveAttributes(ancestorIds);
389 }
390
391 // We are querying in batches because something can't handle WHERE IN queries with sets larger than 999
392 const int querySizeLimit = 999;
393 int mimetypeQueryStart = 0;
394 int attributeQueryStart = 0;
395 std::optional<QueryBuilder> mimeTypeQb;
396 std::optional<QueryBuilder> attributeQb;
397 auto it = mCollections.begin();
398 while (it != mCollections.end()) {
399 const Collection col = it.value();
400
401 QStringList mimeTypes;
402 {
403 // Get new query if necessary
404 if (!mimeTypeQb && mimetypeQueryStart < mimeTypeIds.size()) {
405 const QVariantList ids = mimeTypeIds.mid(mimetypeQueryStart, querySizeLimit);
406 mimetypeQueryStart += querySizeLimit;
407 mimeTypeQb = getMimeTypeQuery(ids);
408 mimeTypeQb->query().next(); // place at first record
409 }
410
411 while (mimeTypeQb && mimeTypeQb->query().isValid() && mimeTypeQb->query().value(0).toLongLong() < col.id()) {
412 if (!mimeTypeQb->query().next()) {
413 break;
414 }
415 }
416 // Advance query while a mimetype for this collection is returned
417 while (mimeTypeQb && mimeTypeQb->query().isValid() && mimeTypeQb->query().value(0).toLongLong() == col.id()) {
418 mimeTypes << mimeTypeQb->query().value(2).toString();
419 if (!mimeTypeQb->query().next()) {
420 break;
421 }
422 }
423 }
424
425 CollectionAttribute::List attributes;
426 {
427 // Get new query if necessary
428 if (!attributeQb && attributeQueryStart < attributeIds.size()) {
429 const QVariantList ids = attributeIds.mid(attributeQueryStart, querySizeLimit);
430 attributeQueryStart += querySizeLimit;
431 attributeQb = getAttributeQuery(ids, QSet<QByteArray>());
432 attributeQb->query().next(); // place at first record
433 }
434
435 while (attributeQb && attributeQb->query().isValid() && attributeQb->query().value(0).toLongLong() < col.id()) {
436 if (!attributeQb->query().next()) {
437 break;
438 }
439 }
440 // Advance query while a mimetype for this collection is returned
441 while (attributeQb && attributeQb->query().isValid() && attributeQb->query().value(0).toLongLong() == col.id()) {
442 auto &attributeQuery = attributeQb->query();
443 CollectionAttribute attr;
444 attr.setType(attributeQuery.value(1).toByteArray());
445 attr.setValue(attributeQuery.value(2).toByteArray());
446 attributes << attr;
447
448 if (!attributeQuery.next()) {
449 break;
450 }
451 }
452 }
453
454 listCollection(col, ancestorsForCollection(col), mimeTypes, attributes);
455 it++;
456 }
457}
458
460{
461 const auto &cmd = Protocol::cmdCast<Protocol::FetchCollectionsCommand>(m_command);
462
463 if (!cmd.resource().isEmpty()) {
464 mResource = Resource::retrieveByName(cmd.resource());
465 if (!mResource.isValid()) {
466 return failureResponse(QStringLiteral("Unknown resource %1").arg(cmd.resource()));
467 }
468 }
469 const QStringList lstMimeTypes = cmd.mimeTypes();
470 for (const QString &mtName : lstMimeTypes) {
471 const MimeType mt = MimeType::retrieveByNameOrCreate(mtName);
472 if (!mt.isValid()) {
473 return failureResponse("Failed to create mimetype record");
474 }
475 mMimeTypes.append(mt.id());
476 }
477
478 mEnabledCollections = cmd.enabled();
479 mCollectionsToSynchronize = cmd.syncPref();
480 mCollectionsToDisplay = cmd.displayPref();
481 mCollectionsToIndex = cmd.indexPref();
482 mIncludeStatistics = cmd.fetchStats();
483
484 int depth = 0;
485 switch (cmd.depth()) {
486 case Protocol::FetchCollectionsCommand::BaseCollection:
487 depth = 0;
488 break;
489 case Protocol::FetchCollectionsCommand::ParentCollection:
490 depth = 1;
491 break;
492 case Protocol::FetchCollectionsCommand::AllCollections:
493 depth = INT_MAX;
494 break;
495 }
496
497 switch (cmd.ancestorsDepth()) {
498 case Protocol::Ancestor::NoAncestor:
499 mAncestorDepth = 0;
500 break;
501 case Protocol::Ancestor::ParentAncestor:
502 mAncestorDepth = 1;
503 break;
504 case Protocol::Ancestor::AllAncestors:
505 mAncestorDepth = INT_MAX;
506 break;
507 }
508 mAncestorAttributes = cmd.ancestorsAttributes();
509
510 Scope scope = cmd.collections();
511 if (!isRootCollection(scope)) {
512 Collection col;
513 if (scope.scope() == Scope::Uid) {
514 col = Collection::retrieveById(scope.uid());
515 } else if (scope.scope() == Scope::Rid) {
517 qb.addValueCondition(Collection::remoteIdFullColumnName(), Query::Equals, scope.rid());
518 qb.addJoin(QueryBuilder::InnerJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
519 if (mCollectionsToSynchronize) {
520 qb.addCondition(filterCondition(Collection::syncPrefFullColumnName()));
521 } else if (mCollectionsToDisplay) {
522 qb.addCondition(filterCondition(Collection::displayPrefFullColumnName()));
523 } else if (mCollectionsToIndex) {
524 qb.addCondition(filterCondition(Collection::indexPrefFullColumnName()));
525 }
526 if (mResource.isValid()) {
527 qb.addValueCondition(Resource::idFullColumnName(), Query::Equals, mResource.id());
528 } else if (connection()->context().resource().isValid()) {
529 qb.addValueCondition(Resource::idFullColumnName(), Query::Equals, connection()->context().resource().id());
530 } else {
531 return failureResponse("Cannot retrieve collection based on remote identifier without a resource context");
532 }
533 if (!qb.exec()) {
534 return failureResponse("Unable to retrieve collection for listing");
535 }
536 Collection::List results = qb.result();
537 if (results.count() != 1) {
538 return failureResponse(QString::number(results.count()) + QStringLiteral(" collections found"));
539 }
540 col = results.first();
541 } else if (scope.scope() == Scope::HierarchicalRid) {
542 if (!connection()->context().resource().isValid()) {
543 return failureResponse("Cannot retrieve collection based on hierarchical remote identifier without a resource context");
544 }
545 col = CollectionQueryHelper::resolveHierarchicalRID(scope.hridChain(), connection()->context().resource().id());
546 } else {
547 return failureResponse("Unexpected error");
548 }
549
550 if (!col.isValid()) {
551 return failureResponse("Collection does not exist");
552 }
553
554 retrieveCollections(col, depth);
555 } else { // Root folder listing
556 if (depth != 0) {
557 retrieveCollections(Collection(), depth);
558 }
559 }
560
561 return successResponse<Protocol::FetchCollectionsResponse>();
562}
Represents a collection of PIM items.
Definition collection.h:62
qint64 Id
Describes the unique id type.
Definition collection.h:79
bool parseStream() override
Parse and handle the IMAP message using the streaming parser.
virtual void activeCachePolicy(Collection &col)
Determines the active cache policy for this Collection.
static Protocol::FetchCollectionsResponse fetchCollectionsResponse(AkonadiServer &akonadi, const Collection &col)
Returns the protocol representation of the given collection.
The handler interfaces describes an entity capable of handling an AkonadiIMAP command.
Definition handler.h:32
Helper class to construct arbitrary SQL queries.
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
void addGroupColumn(const QString &column)
Add a GROUP BY column.
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
bool exec()
Executes the query, returns true on success.
void addCondition(const Query::Condition &condition, ConditionType type=WhereCondition)
Add a WHERE condition.
@ InnerJoin
NOTE: only supported for UPDATE and SELECT queries.
@ LeftJoin
NOTE: only supported for SELECT queries.
Represents a WHERE condition tree.
Definition query.h:62
Helper class for creating and executing database SELECT queries.
QList< T > result()
Returns the result of this SELECT query.
Q_SCRIPTABLE Q_NOREPLY void start()
Collection resolveHierarchicalRID(const QList< Scope::HRID > &hridChain, Resource::Id resId)
Retrieve the collection referred to by the given hierarchical RID chain.
Helper integration between Akonadi and Qt.
VehicleSection::Type type(QStringView coachNumber, QStringView coachClassification)
void append(QList< T > &&value)
bool contains(const AT &value) const const
qsizetype count() const const
T & first()
bool isEmpty() const const
void prepend(parameter_type value)
void reserve(qsizetype size)
qsizetype size() const const
T value(qsizetype i) const const
iterator end()
iterator find(const Key &key, const T &value)
iterator insert(const Key &key, const T &value)
void clear()
iterator insert(const T &value)
bool isEmpty() const const
qsizetype size() const const
void push(const T &t)
QString number(double n, char format, int precision)
QFuture< void > filter(QThreadPool *pool, Sequence &sequence, KeepFunctor &&filterFunction)
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Mon Nov 4 2024 16:31:59 by doxygen 1.12.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.