Akonadi

collectionfetchhandler.cpp
1 /*
2  SPDX-FileCopyrightText: 2007 Volker Krause <[email protected]>
3 
4  SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6 
7 #include "collectionfetchhandler.h"
8 #include "akonadiserver_debug.h"
9 
10 #include "connection.h"
11 #include "handlerhelper.h"
12 #include "storage/collectionqueryhelper.h"
13 #include "storage/datastore.h"
14 #include "storage/selectquerybuilder.h"
15 
16 #include <private/scope_p.h>
17 
18 using namespace Akonadi;
19 using namespace Akonadi::Server;
20 
21 template<typename T> static bool intersect(const QVector<typename T::Id> &l1, const QVector<T> &l2)
22 {
23  for (const T &e2 : l2) {
24  if (l1.contains(e2.id())) {
25  return true;
26  }
27  }
28  return false;
29 }
30 
31 CollectionFetchHandler::CollectionFetchHandler(AkonadiServer &akonadi)
32  : Handler(akonadi)
33 {
34 }
35 
36 QStack<Collection> CollectionFetchHandler::ancestorsForCollection(const Collection &col)
37 {
38  if (mAncestorDepth <= 0) {
39  return QStack<Collection>();
40  }
41  QStack<Collection> ancestors;
42  Collection parent = col;
43  for (int i = 0; i < mAncestorDepth; ++i) {
44  if (parent.parentId() == 0) {
45  break;
46  }
47  if (mAncestors.contains(parent.parentId())) {
48  parent = mAncestors.value(parent.parentId());
49  } else {
50  parent = mCollections.value(parent.parentId());
51  }
52  if (!parent.isValid()) {
53  qCWarning(AKONADISERVER_LOG) << "Found an invalid parent in ancestors of Collection" << col.name() << "(ID:" << col.id() << ")";
54  throw HandlerException("Found invalid parent in ancestors");
55  }
56  ancestors.prepend(parent);
57  }
58  return ancestors;
59 }
60 
61 CollectionAttribute::List CollectionFetchHandler::getAttributes(const Collection &col, const QSet<QByteArray> &filter)
62 {
63  CollectionAttribute::List attributes;
64  auto it = mCollectionAttributes.find(col.id());
65  while (it != mCollectionAttributes.end() && it.key() == col.id()) {
66  if (filter.isEmpty() || filter.contains(it.value().type())) {
67  attributes << it.value();
68  }
69  ++it;
70  }
71 
72  {
73  CollectionAttribute attr;
74  attr.setType(AKONADI_PARAM_ENABLED);
75  attr.setValue(col.enabled() ? "TRUE" : "FALSE");
76  attributes << attr;
77  }
78 
79  return attributes;
80 }
81 
82 void CollectionFetchHandler::listCollection(const Collection &root,
83  const QStack<Collection> &ancestors,
84  const QStringList &mimeTypes,
85  const CollectionAttribute::List &attributes)
86 {
87  QStack<CollectionAttribute::List> ancestorAttributes;
88  // backwards compatibility, collectionToByteArray will automatically fall-back to id + remoteid
89  if (!mAncestorAttributes.isEmpty()) {
90  ancestorAttributes.reserve(ancestors.size());
91  for (const Collection &col : ancestors) {
92  ancestorAttributes.push(getAttributes(col, mAncestorAttributes));
93  }
94  }
95 
96  // write out collection details
97  Collection dummy = root;
98  storageBackend()->activeCachePolicy(dummy);
99 
100  sendResponse(
101  HandlerHelper::fetchCollectionsResponse(akonadi(), dummy, attributes, mIncludeStatistics, mAncestorDepth, ancestors, ancestorAttributes, mimeTypes));
102 }
103 
104 static Query::Condition filterCondition(const QString &column)
105 {
106  Query::Condition orCondition(Query::Or);
107  orCondition.addValueCondition(column, Query::Equals, static_cast<int>(Collection::True));
108  Query::Condition andCondition(Query::And);
109  andCondition.addValueCondition(column, Query::Equals, static_cast<int>(Collection::Undefined));
110  andCondition.addValueCondition(Collection::enabledFullColumnName(), Query::Equals, true);
111  orCondition.addCondition(andCondition);
112  return orCondition;
113 }
114 
115 bool CollectionFetchHandler::checkFilterCondition(const Collection &col) const
116 {
117  // Don't include the collection when only looking for enabled collections
118  if (mEnabledCollections && !col.enabled()) {
119  return false;
120  }
121  // Don't include the collection when only looking for collections to display/index/sync
122  if (mCollectionsToDisplay && (((col.displayPref() == Collection::Undefined) && !col.enabled()) || (col.displayPref() == Collection::False))) {
123  return false;
124  }
125  if (mCollectionsToIndex && (((col.indexPref() == Collection::Undefined) && !col.enabled()) || (col.indexPref() == Collection::False))) {
126  return false;
127  }
128  // Single collection sync will still work since that is using a base fetch
129  if (mCollectionsToSynchronize && (((col.syncPref() == Collection::Undefined) && !col.enabled()) || (col.syncPref() == Collection::False))) {
130  return false;
131  }
132  return true;
133 }
134 
135 static QSqlQuery getAttributeQuery(const QVariantList &ids, const QSet<QByteArray> &requestedAttributes)
136 {
137  QueryBuilder qb(CollectionAttribute::tableName());
138 
139  qb.addValueCondition(CollectionAttribute::collectionIdFullColumnName(), Query::In, ids);
140 
141  qb.addColumn(CollectionAttribute::collectionIdFullColumnName());
142  qb.addColumn(CollectionAttribute::typeFullColumnName());
143  qb.addColumn(CollectionAttribute::valueFullColumnName());
144 
145  if (!requestedAttributes.isEmpty()) {
146  QVariantList attributes;
147  attributes.reserve(requestedAttributes.size());
148  for (const QByteArray &type : requestedAttributes) {
149  attributes << type;
150  }
151  qb.addValueCondition(CollectionAttribute::typeFullColumnName(), Query::In, attributes);
152  }
153 
154  qb.addSortColumn(CollectionAttribute::collectionIdFullColumnName(), Query::Ascending);
155 
156  if (!qb.exec()) {
157  throw HandlerException("Unable to retrieve attributes for listing");
158  }
159  return qb.query();
160 }
161 
162 void CollectionFetchHandler::retrieveAttributes(const QVariantList &collectionIds)
163 {
164  // We are querying for the attributes in batches because something can't handle WHERE IN queries with sets larger than 999
165  int start = 0;
166  const int size = 999;
167  while (start < collectionIds.size()) {
168  const QVariantList ids = collectionIds.mid(start, size);
169  QSqlQuery attributeQuery = getAttributeQuery(ids, mAncestorAttributes);
170  while (attributeQuery.next()) {
171  CollectionAttribute attr;
172  attr.setType(attributeQuery.value(1).toByteArray());
173  attr.setValue(attributeQuery.value(2).toByteArray());
174  // qCDebug(AKONADISERVER_LOG) << "found attribute " << attr.type() << attr.value();
175  mCollectionAttributes.insert(attributeQuery.value(0).toLongLong(), attr);
176  }
177  attributeQuery.finish();
178  start += size;
179  }
180 }
181 
182 static QSqlQuery getMimeTypeQuery(const QVariantList &ids)
183 {
184  QueryBuilder qb(CollectionMimeTypeRelation::tableName());
185 
186  qb.addJoin(QueryBuilder::LeftJoin, MimeType::tableName(), MimeType::idFullColumnName(), CollectionMimeTypeRelation::rightFullColumnName());
187  qb.addValueCondition(CollectionMimeTypeRelation::leftFullColumnName(), Query::In, ids);
188 
189  qb.addColumn(CollectionMimeTypeRelation::leftFullColumnName());
190  qb.addColumn(CollectionMimeTypeRelation::rightFullColumnName());
191  qb.addColumn(MimeType::nameFullColumnName());
192  qb.addSortColumn(CollectionMimeTypeRelation::leftFullColumnName(), Query::Ascending);
193 
194  if (!qb.exec()) {
195  throw HandlerException("Unable to retrieve mimetypes for listing");
196  }
197  return qb.query();
198 }
199 
200 void CollectionFetchHandler::retrieveCollections(const Collection &topParent, int depth)
201 {
202  /*
203  * Retrieval of collections:
204  * The aim is to reduce the amount of queries as much as possible, as this has the largest performance impact for large queries.
205  * * First all collections that match the given criteria are queried
206  * * We then filter the false positives:
207  * ** all collections out that are not part of the tree we asked for are filtered
208  * * Finally we complete the tree by adding missing collections
209  *
210  * Mimetypes and attributes are also retrieved in single queries to avoid spawning two queries per collection (the N+1 problem).
211  * Note that we're not querying attributes and mimetypes for the collections that are only included to complete the tree,
212  * this results in no items being queried for those collections.
213  */
214 
215  const qint64 parentId = topParent.isValid() ? topParent.id() : 0;
216  {
218 
219  if (depth == 0) {
220  qb.addValueCondition(Collection::idFullColumnName(), Query::Equals, parentId);
221  } else if (depth == 1) {
222  if (topParent.isValid()) {
223  qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Equals, parentId);
224  } else {
225  qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant());
226  }
227  } else {
228  if (topParent.isValid()) {
229  qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, topParent.resourceId());
230  } else {
231  // Gimme gimme gimme...everything!
232  }
233  }
234 
235  // Base listings should succeed always
236  if (depth != 0) {
237  if (mCollectionsToSynchronize) {
238  qb.addCondition(filterCondition(Collection::syncPrefFullColumnName()));
239  } else if (mCollectionsToDisplay) {
240  qb.addCondition(filterCondition(Collection::displayPrefFullColumnName()));
241  } else if (mCollectionsToIndex) {
242  qb.addCondition(filterCondition(Collection::indexPrefFullColumnName()));
243  } else if (mEnabledCollections) {
244  qb.addValueCondition(Collection::enabledFullColumnName(), Query::Equals, true);
245  }
246  if (mResource.isValid()) {
247  qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, mResource.id());
248  }
249 
250  if (!mMimeTypes.isEmpty()) {
251  qb.addJoin(QueryBuilder::LeftJoin,
252  CollectionMimeTypeRelation::tableName(),
253  CollectionMimeTypeRelation::leftColumn(),
254  Collection::idFullColumnName());
255  QVariantList mimeTypeFilter;
256  mimeTypeFilter.reserve(mMimeTypes.size());
257  for (MimeType::Id mtId : std::as_const(mMimeTypes)) {
258  mimeTypeFilter << mtId;
259  }
260  qb.addValueCondition(CollectionMimeTypeRelation::rightColumn(), Query::In, mimeTypeFilter);
261  qb.addGroupColumn(Collection::idFullColumnName());
262  }
263  }
264 
265  if (!qb.exec()) {
266  throw HandlerException("Unable to retrieve collection for listing");
267  }
268  const auto result{qb.result()};
269  for (const Collection &col : result) {
270  mCollections.insert(col.id(), col);
271  }
272  }
273 
274  // Post filtering that we couldn't do as part of the sql query
275  if (depth > 0) {
276  auto it = mCollections.begin();
277  while (it != mCollections.end()) {
278  if (topParent.isValid()) {
279  // Check that each collection is linked to the root collection
280  bool foundParent = false;
281  // We iterate over parents to link it to topParent if possible
282  Collection::Id id = it->parentId();
283  while (id > 0) {
284  if (id == parentId) {
285  foundParent = true;
286  break;
287  }
288  Collection col = mCollections.value(id);
289  if (!col.isValid()) {
290  col = Collection::retrieveById(id);
291  }
292  id = col.parentId();
293  }
294  if (!foundParent) {
295  it = mCollections.erase(it);
296  continue;
297  }
298  }
299  ++it;
300  }
301  }
302 
303  QVariantList mimeTypeIds;
304  QVariantList attributeIds;
305  QVariantList ancestorIds;
306  const auto collectionSize{mCollections.size()};
307  mimeTypeIds.reserve(collectionSize);
308  attributeIds.reserve(collectionSize);
309  // We'd only require the non-leaf collections, but we don't know which those are, so we take all.
310  ancestorIds.reserve(collectionSize);
311  for (auto it = mCollections.cbegin(), end = mCollections.cend(); it != end; ++it) {
312  mimeTypeIds << it.key();
313  attributeIds << it.key();
314  ancestorIds << it.key();
315  }
316 
317  if (mAncestorDepth > 0 && topParent.isValid()) {
318  // unless depth is 0 the base collection is not part of the listing
319  mAncestors.insert(topParent.id(), topParent);
320  ancestorIds << topParent.id();
321  // We need to retrieve additional ancestors to what we already have in the tree
322  Collection parent = topParent;
323  for (int i = 0; i < mAncestorDepth; ++i) {
324  if (parent.parentId() == 0) {
325  break;
326  }
327  parent = parent.parent();
328  mAncestors.insert(parent.id(), parent);
329  // We also require the attributes
330  ancestorIds << parent.id();
331  }
332  }
333 
334  QSet<qint64> missingCollections;
335  if (depth > 0) {
336  for (const Collection &col : std::as_const(mCollections)) {
337  if (col.parentId() != parentId && !mCollections.contains(col.parentId())) {
338  missingCollections.insert(col.parentId());
339  }
340  }
341  }
342 
343  /*
344  QSet<qint64> knownIds;
345  for (const Collection &col : mCollections) {
346  knownIds.insert(col.id());
347  }
348  qCDebug(AKONADISERVER_LOG) << "HAS:" << knownIds;
349  qCDebug(AKONADISERVER_LOG) << "MISSING:" << missingCollections;
350  */
351 
352  // Fetch missing collections that are part of the tree
353  while (!missingCollections.isEmpty()) {
355  QVariantList ids;
356  ids.reserve(missingCollections.size());
357  for (qint64 id : std::as_const(missingCollections)) {
358  ids << id;
359  }
360  qb.addValueCondition(Collection::idFullColumnName(), Query::In, ids);
361  if (!qb.exec()) {
362  throw HandlerException("Unable to retrieve collections for listing");
363  }
364 
365  missingCollections.clear();
366  const auto missingCols = qb.result();
367  for (const Collection &missingCol : missingCols) {
368  mCollections.insert(missingCol.id(), missingCol);
369  ancestorIds << missingCol.id();
370  attributeIds << missingCol.id();
371  mimeTypeIds << missingCol.id();
372  // We have to do another round if the parents parent is missing
373  if (missingCol.parentId() != parentId && !mCollections.contains(missingCol.parentId())) {
374  missingCollections.insert(missingCol.parentId());
375  }
376  }
377  }
378 
379  // Since we don't know when we'll need the ancestor attributes, we have to fetch them all together.
380  // The alternative would be to query for each collection which would reintroduce the N+1 query performance problem.
381  if (!mAncestorAttributes.isEmpty()) {
382  retrieveAttributes(ancestorIds);
383  }
384 
385  // We are querying in batches because something can't handle WHERE IN queries with sets larger than 999
386  const int querySizeLimit = 999;
387  int mimetypeQueryStart = 0;
388  int attributeQueryStart = 0;
389  QSqlQuery mimeTypeQuery(storageBackend()->database());
390  QSqlQuery attributeQuery(storageBackend()->database());
391  auto it = mCollections.begin();
392  while (it != mCollections.end()) {
393  const Collection col = it.value();
394 
396  {
397  // Get new query if necessary
398  if (!mimeTypeQuery.isValid() && mimetypeQueryStart < mimeTypeIds.size()) {
399  const QVariantList ids = mimeTypeIds.mid(mimetypeQueryStart, querySizeLimit);
400  mimetypeQueryStart += querySizeLimit;
401  mimeTypeQuery = getMimeTypeQuery(ids);
402  mimeTypeQuery.next(); // place at first record
403  }
404 
405  while (mimeTypeQuery.isValid() && mimeTypeQuery.value(0).toLongLong() < col.id()) {
406  if (!mimeTypeQuery.next()) {
407  break;
408  }
409  }
410  // Advance query while a mimetype for this collection is returned
411  while (mimeTypeQuery.isValid() && mimeTypeQuery.value(0).toLongLong() == col.id()) {
412  mimeTypes << mimeTypeQuery.value(2).toString();
413  if (!mimeTypeQuery.next()) {
414  break;
415  }
416  }
417  }
418 
419  CollectionAttribute::List attributes;
420  {
421  // Get new query if necessary
422  if (!attributeQuery.isValid() && attributeQueryStart < attributeIds.size()) {
423  const QVariantList ids = attributeIds.mid(attributeQueryStart, querySizeLimit);
424  attributeQueryStart += querySizeLimit;
425  attributeQuery = getAttributeQuery(ids, QSet<QByteArray>());
426  attributeQuery.next(); // place at first record
427  }
428 
429  while (attributeQuery.isValid() && attributeQuery.value(0).toLongLong() < col.id()) {
430  if (!attributeQuery.next()) {
431  break;
432  }
433  }
434  // Advance query while a mimetype for this collection is returned
435  while (attributeQuery.isValid() && attributeQuery.value(0).toLongLong() == col.id()) {
436  CollectionAttribute attr;
437  attr.setType(attributeQuery.value(1).toByteArray());
438  attr.setValue(attributeQuery.value(2).toByteArray());
439  attributes << attr;
440 
441  if (!attributeQuery.next()) {
442  break;
443  }
444  }
445  }
446 
447  listCollection(col, ancestorsForCollection(col), mimeTypes, attributes);
448  it++;
449  }
450  attributeQuery.finish();
451  mimeTypeQuery.finish();
452 }
453 
454 bool CollectionFetchHandler::parseStream()
455 {
456  const auto &cmd = Protocol::cmdCast<Protocol::FetchCollectionsCommand>(m_command);
457 
458  if (!cmd.resource().isEmpty()) {
459  mResource = Resource::retrieveByName(cmd.resource());
460  if (!mResource.isValid()) {
461  return failureResponse("Unknown resource");
462  }
463  }
464  const QStringList lstMimeTypes = cmd.mimeTypes();
465  for (const QString &mtName : lstMimeTypes) {
466  const MimeType mt = MimeType::retrieveByNameOrCreate(mtName);
467  if (!mt.isValid()) {
468  return failureResponse("Failed to create mimetype record");
469  }
470  mMimeTypes.append(mt.id());
471  }
472 
473  mEnabledCollections = cmd.enabled();
474  mCollectionsToSynchronize = cmd.syncPref();
475  mCollectionsToDisplay = cmd.displayPref();
476  mCollectionsToIndex = cmd.indexPref();
477  mIncludeStatistics = cmd.fetchStats();
478 
479  int depth = 0;
480  switch (cmd.depth()) {
481  case Protocol::FetchCollectionsCommand::BaseCollection:
482  depth = 0;
483  break;
484  case Protocol::FetchCollectionsCommand::ParentCollection:
485  depth = 1;
486  break;
487  case Protocol::FetchCollectionsCommand::AllCollections:
488  depth = INT_MAX;
489  break;
490  }
491 
492  switch (cmd.ancestorsDepth()) {
493  case Protocol::Ancestor::NoAncestor:
494  mAncestorDepth = 0;
495  break;
496  case Protocol::Ancestor::ParentAncestor:
497  mAncestorDepth = 1;
498  break;
499  case Protocol::Ancestor::AllAncestors:
500  mAncestorDepth = INT_MAX;
501  break;
502  }
503  mAncestorAttributes = cmd.ancestorsAttributes();
504 
505  Scope scope = cmd.collections();
506  if (!scope.isEmpty()) { // not root
507  Collection col;
508  if (scope.scope() == Scope::Uid) {
509  col = Collection::retrieveById(scope.uid());
510  } else if (scope.scope() == Scope::Rid) {
512  qb.addValueCondition(Collection::remoteIdFullColumnName(), Query::Equals, scope.rid());
513  qb.addJoin(QueryBuilder::InnerJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
514  if (mCollectionsToSynchronize) {
515  qb.addCondition(filterCondition(Collection::syncPrefFullColumnName()));
516  } else if (mCollectionsToDisplay) {
517  qb.addCondition(filterCondition(Collection::displayPrefFullColumnName()));
518  } else if (mCollectionsToIndex) {
519  qb.addCondition(filterCondition(Collection::indexPrefFullColumnName()));
520  }
521  if (mResource.isValid()) {
522  qb.addValueCondition(Resource::idFullColumnName(), Query::Equals, mResource.id());
523  } else if (connection()->context().resource().isValid()) {
524  qb.addValueCondition(Resource::idFullColumnName(), Query::Equals, connection()->context().resource().id());
525  } else {
526  return failureResponse("Cannot retrieve collection based on remote identifier without a resource context");
527  }
528  if (!qb.exec()) {
529  return failureResponse("Unable to retrieve collection for listing");
530  }
531  Collection::List results = qb.result();
532  if (results.count() != 1) {
533  return failureResponse(QString::number(results.count()) + QStringLiteral(" collections found"));
534  }
535  col = results.first();
536  } else if (scope.scope() == Scope::HierarchicalRid) {
537  if (!connection()->context().resource().isValid()) {
538  return failureResponse("Cannot retrieve collection based on hierarchical remote identifier without a resource context");
539  }
540  col = CollectionQueryHelper::resolveHierarchicalRID(scope.hridChain(), connection()->context().resource().id());
541  } else {
542  return failureResponse("Unexpected error");
543  }
544 
545  if (!col.isValid()) {
546  return failureResponse("Collection does not exist");
547  }
548 
549  retrieveCollections(col, depth);
550  } else { // Root folder listing
551  if (depth != 0) {
552  retrieveCollections(Collection(), depth);
553  }
554  }
555 
556  return successResponse<Protocol::FetchCollectionsResponse>();
557 }
QVariant value(int index) const const
void prepend(T &&value)
void addGroupColumn(const QString &column)
Add a GROUP BY column.
QString number(int n, int base)
void insert(int i, T &&value)
Q_SCRIPTABLE Q_NOREPLY void start()
Represents a collection of PIM items.
Definition: collection.h:61
bool isValid() const const
QByteArray toByteArray() const const
qlonglong toLongLong(bool *ok) const const
T & first()
int size() const const
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
T value(int i) const const
QList< T > mid(int pos, int length) const const
QVector< T > result()
Returns the result of this SELECT query.
void finish()
QFuture< void > filter(Sequence &sequence, KeepFunctor filterFunction)
bool contains(const T &value) const const
bool next()
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
void reserve(int size)
bool exec()
Executes the query, returns true on success.
VehicleSection::Type type(QStringView coachNumber, QStringView coachClassification)
void push(const T &t)
Helper class for creating and executing database SELECT queries.
The handler interfaces describes an entity capable of handling an AkonadiIMAP command.
Definition: handler.h:39
void clear()
QStringList mimeTypes(Mode mode=Writing)
QSet::iterator insert(const T &value)
int count(const T &value) const const
int size() const const
Represents a WHERE condition tree.
Definition: query.h:61
void addCondition(const Query::Condition &condition, ConditionType type=WhereCondition)
Add a WHERE condition.
qint64 Id
Describes the unique id type.
Definition: collection.h:79
T value(int i) const const
bool isEmpty() const const
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:31
Helper integration between Akonadi and Qt.
This file is part of the KDE documentation.
Documentation copyright © 1996-2022 The KDE developers.
Generated on Sat Jul 2 2022 06:41:47 by doxygen 1.8.17 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.