Akonadi

storagejanitor.cpp
1 /*
2  SPDX-FileCopyrightText: 2011 Volker Krause <[email protected]>
3 
4  SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6 
7 #include "storagejanitor.h"
8 #include "agentmanagerinterface.h"
9 #include "akonadi.h"
10 #include "akonadiserver_debug.h"
11 #include "entities.h"
12 #include "resourcemanager.h"
13 #include "search/searchmanager.h"
14 #include "search/searchrequest.h"
15 #include "storage/collectionstatistics.h"
16 #include "storage/datastore.h"
17 #include "storage/dbconfig.h"
18 #include "storage/parthelper.h"
19 #include "storage/queryhelper.h"
20 #include "storage/selectquerybuilder.h"
21 #include "storage/transaction.h"
22 
23 #include <private/dbus_p.h>
24 #include <private/externalpartstorage_p.h>
25 #include <private/imapset_p.h>
26 #include <private/protocol_p.h>
27 #include <private/standarddirs_p.h>
28 
29 #include <QDateTime>
30 #include <QDir>
31 #include <QDirIterator>
32 #include <QSqlError>
33 #include <QSqlQuery>
34 #include <QStringBuilder>
35 
36 #include <algorithm>
37 
38 using namespace Akonadi;
39 using namespace Akonadi::Server;
40 
41 StorageJanitor::StorageJanitor(AkonadiServer &akonadi)
42  : AkThread(QStringLiteral("StorageJanitor"), QThread::IdlePriority)
43  , m_lostFoundCollectionId(-1)
44  , m_akonadi(akonadi)
45 {
46 }
47 
48 StorageJanitor::~StorageJanitor()
49 {
50  quitThread();
51 }
52 
53 void StorageJanitor::init()
54 {
55  AkThread::init();
56 
58  conn.registerService(DBus::serviceName(DBus::StorageJanitor));
59  conn.registerObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH),
60  this,
62 }
63 
64 void StorageJanitor::quit()
65 {
67  conn.unregisterObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), QDBusConnection::UnregisterTree);
68  conn.unregisterService(DBus::serviceName(DBus::StorageJanitor));
69  conn.disconnectFromBus(conn.name());
70 
71  // Make sure all children are deleted within context of this thread
72  qDeleteAll(children());
73 
74  AkThread::quit();
75 }
76 
77 void StorageJanitor::check() // implementation of `akonadictl fsck`
78 {
79  m_lostFoundCollectionId = -1; // start with a fresh one each time
80 
81  inform("Looking for resources in the DB not matching a configured resource...");
82  findOrphanedResources();
83 
84  inform("Looking for collections not belonging to a valid resource...");
85  findOrphanedCollections();
86 
87  inform("Checking collection tree consistency...");
88  const Collection::List cols = Collection::retrieveAll();
89  std::for_each(cols.begin(), cols.end(), [this](const Collection &col) {
90  checkPathToRoot(col);
91  });
92 
93  inform("Looking for items not belonging to a valid collection...");
94  findOrphanedItems();
95 
96  inform("Looking for item parts not belonging to a valid item...");
97  findOrphanedParts();
98 
99  inform("Looking for item flags not belonging to a valid item...");
100  findOrphanedPimItemFlags();
101 
102  inform("Looking for overlapping external parts...");
103  findOverlappingParts();
104 
105  inform("Verifying external parts...");
106  verifyExternalParts();
107 
108  inform("Checking size threshold changes...");
109  checkSizeTreshold();
110 
111  inform("Looking for dirty objects...");
112  findDirtyObjects();
113 
114  inform("Looking for rid-duplicates not matching the content mime-type of the parent collection");
115  findRIDDuplicates();
116 
117  inform("Migrating parts to new cache hierarchy...");
118  migrateToLevelledCacheHierarchy();
119 
120  inform("Checking search index consistency...");
121  findOrphanSearchIndexEntries();
122 
123  inform("Flushing collection statistics memory cache...");
124  m_akonadi.collectionStatistics().expireCache();
125 
126  inform("Making sure virtual search resource and collections exist");
127  ensureSearchCollection();
128 
129  /* TODO some ideas for further checks:
130  * the collection tree is non-cyclic
131  * content type constraints of collections are not violated
132  * find unused flags
133  * find unused mimetypes
134  * check for dead entries in relation tables
135  * check if part size matches file size
136  */
137 
138  inform("Consistency check done.");
139 
140  Q_EMIT done();
141 }
142 
143 qint64 StorageJanitor::lostAndFoundCollection()
144 {
145  if (m_lostFoundCollectionId > 0) {
146  return m_lostFoundCollectionId;
147  }
148 
149  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR LOST+FOUND"));
150  Resource lfRes = Resource::retrieveByName(QStringLiteral("akonadi_lost+found_resource"));
151  if (!lfRes.isValid()) {
152  lfRes.setName(QStringLiteral("akonadi_lost+found_resource"));
153  if (!lfRes.insert()) {
154  qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found resource!";
155  }
156  }
157 
158  Collection lfRoot;
160  qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, lfRes.id());
161  qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant());
162  if (!qb.exec()) {
163  qCCritical(AKONADISERVER_LOG) << "Failed to query top level collections";
164  return -1;
165  }
166  const Collection::List cols = qb.result();
167  if (cols.size() > 1) {
168  qCCritical(AKONADISERVER_LOG) << "More than one top-level lost+found collection!?";
169  } else if (cols.size() == 1) {
170  lfRoot = cols.first();
171  } else {
172  lfRoot.setName(QStringLiteral("lost+found"));
173  lfRoot.setResourceId(lfRes.id());
174  lfRoot.setCachePolicyLocalParts(QStringLiteral("ALL"));
175  lfRoot.setCachePolicyCacheTimeout(-1);
176  lfRoot.setCachePolicyInherit(false);
177  if (!lfRoot.insert()) {
178  qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found root.";
179  }
180  DataStore::self()->notificationCollector()->collectionAdded(lfRoot, lfRes.name().toUtf8());
181  }
182 
183  Collection lfCol;
184  lfCol.setName(QDateTime::currentDateTime().toString(QStringLiteral("yyyy-MM-dd hh:mm:ss")));
185  lfCol.setResourceId(lfRes.id());
186  lfCol.setParentId(lfRoot.id());
187  if (!lfCol.insert()) {
188  qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found collection!";
189  }
190 
191  const auto retrieveAll = MimeType::retrieveAll();
192  for (const MimeType &mt : retrieveAll) {
193  lfCol.addMimeType(mt);
194  }
195 
196  DataStore::self()->notificationCollector()->collectionAdded(lfCol, lfRes.name().toUtf8());
197 
198  transaction.commit();
199  m_lostFoundCollectionId = lfCol.id();
200  return m_lostFoundCollectionId;
201 }
202 
203 void StorageJanitor::findOrphanedResources()
204 {
206  OrgFreedesktopAkonadiAgentManagerInterface iface(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus(), this);
207  if (!iface.isValid()) {
208  inform(QStringLiteral("ERROR: Couldn't talk to %1").arg(DBus::Control));
209  return;
210  }
211  const QStringList knownResources = iface.agentInstances();
212  if (knownResources.isEmpty()) {
213  inform(QStringLiteral("ERROR: no known resources. This must be a mistake?"));
214  return;
215  }
216  qbres.addValueCondition(Resource::nameFullColumnName(), Query::NotIn, QVariant(knownResources));
217  qbres.addValueCondition(Resource::idFullColumnName(), Query::NotEquals, 1); // skip akonadi_search_resource
218  if (!qbres.exec()) {
219  inform("Failed to query known resources, skipping test");
220  return;
221  }
222  // qCDebug(AKONADISERVER_LOG) << "SQL:" << qbres.query().lastQuery();
223  const Resource::List orphanResources = qbres.result();
224  const int orphanResourcesSize(orphanResources.size());
225  if (orphanResourcesSize > 0) {
226  QStringList resourceNames;
227  resourceNames.reserve(orphanResourcesSize);
228  for (const Resource &resource : orphanResources) {
229  resourceNames.append(resource.name());
230  }
231  inform(QStringLiteral("Found %1 orphan resources: %2").arg(orphanResourcesSize).arg(resourceNames.join(QLatin1Char(','))));
232  for (const QString &resourceName : std::as_const(resourceNames)) {
233  inform(QStringLiteral("Removing resource %1").arg(resourceName));
234  m_akonadi.resourceManager().removeResourceInstance(resourceName);
235  }
236  }
237 }
238 
239 void StorageJanitor::findOrphanedCollections()
240 {
242  qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
243  qb.addValueCondition(Resource::idFullColumnName(), Query::Is, QVariant());
244 
245  if (!qb.exec()) {
246  inform("Failed to query orphaned collections, skipping test");
247  return;
248  }
249  const Collection::List orphans = qb.result();
250  if (!orphans.isEmpty()) {
251  inform(QLatin1String("Found ") + QString::number(orphans.size()) + QLatin1String(" orphan collections."));
252  // TODO: attach to lost+found resource
253  }
254 }
255 
256 void StorageJanitor::checkPathToRoot(const Collection &col)
257 {
258  if (col.parentId() == 0) {
259  return;
260  }
261  const Collection parent = col.parent();
262  if (!parent.isValid()) {
263  inform(QLatin1String("Collection \"") + col.name() + QLatin1String("\" (id: ") + QString::number(col.id()) + QLatin1String(") has no valid parent."));
264  // TODO fix that by attaching to a top-level lost+found folder
265  return;
266  }
267 
268  if (col.resourceId() != parent.resourceId()) {
269  inform(QLatin1String("Collection \"") + col.name() + QLatin1String("\" (id: ") + QString::number(col.id())
270  + QLatin1String(") belongs to a different resource than its parent."));
271  // can/should we actually fix that?
272  }
273 
274  checkPathToRoot(parent);
275 }
276 
277 void StorageJanitor::findOrphanedItems()
278 {
280  qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
281  qb.addValueCondition(Collection::idFullColumnName(), Query::Is, QVariant());
282  if (!qb.exec()) {
283  inform("Failed to query orphaned items, skipping test");
284  return;
285  }
286  const PimItem::List orphans = qb.result();
287  if (!orphans.isEmpty()) {
288  inform(QLatin1String("Found ") + QString::number(orphans.size()) + QLatin1String(" orphan items."));
289  // Attach to lost+found collection
290  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR ORPHANS"));
291  QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
292  qint64 col = lostAndFoundCollection();
293  if (col == -1) {
294  return;
295  }
296  qb.setColumnValue(PimItem::collectionIdColumn(), col);
297  QVector<ImapSet::Id> imapIds;
298  imapIds.reserve(orphans.count());
299  for (const PimItem &item : std::as_const(orphans)) {
300  imapIds.append(item.id());
301  }
302  ImapSet set;
303  set.add(imapIds);
304  QueryHelper::setToQuery(set, PimItem::idFullColumnName(), qb);
305  if (qb.exec() && transaction.commit()) {
306  inform(QLatin1String("Moved orphan items to collection ") + QString::number(col));
307  } else {
308  inform(QLatin1String("Error moving orphan items to collection ") + QString::number(col) + QLatin1String(" : ") + qb.query().lastError().text());
309  }
310  }
311 }
312 
313 void StorageJanitor::findOrphanedParts()
314 {
316  qb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
317  qb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
318  if (!qb.exec()) {
319  inform("Failed to query orphaned parts, skipping test");
320  return;
321  }
322  const Part::List orphans = qb.result();
323  if (!orphans.isEmpty()) {
324  inform(QLatin1String("Found ") + QString::number(orphans.size()) + QLatin1String(" orphan parts."));
325  // TODO: create lost+found items for those? delete?
326  }
327 }
328 
329 void StorageJanitor::findOrphanedPimItemFlags()
330 {
331  QueryBuilder sqb(PimItemFlagRelation::tableName(), QueryBuilder::Select);
332  sqb.addColumn(PimItemFlagRelation::leftFullColumnName());
333  sqb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), PimItemFlagRelation::leftFullColumnName(), PimItem::idFullColumnName());
334  sqb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
335  if (!sqb.exec()) {
336  inform("Failed to query orphaned item flags, skipping test");
337  return;
338  }
339  QVector<ImapSet::Id> imapIds;
340  int count = 0;
341  while (sqb.query().next()) {
342  ++count;
343  imapIds.append(sqb.query().value(0).toInt());
344  }
345  sqb.query().finish();
346  if (count > 0) {
347  ImapSet set;
348  set.add(imapIds);
349  QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete);
350  QueryHelper::setToQuery(set, PimItemFlagRelation::leftFullColumnName(), qb);
351  if (!qb.exec()) {
352  qCCritical(AKONADISERVER_LOG) << "Error:" << qb.query().lastError().text();
353  return;
354  }
355 
356  inform(QLatin1String("Found and deleted ") + QString::number(count) + QLatin1String(" orphan pim item flags."));
357  }
358 }
359 
360 void StorageJanitor::findOverlappingParts()
361 {
362  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
363  qb.addColumn(Part::dataColumn());
364  qb.addColumn(QLatin1String("count(") + Part::idColumn() + QLatin1String(") as cnt"));
365  qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
366  qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
367  qb.addGroupColumn(Part::dataColumn());
368  qb.addValueCondition(QLatin1String("count(") + Part::idColumn() + QLatin1String(")"), Query::Greater, 1, QueryBuilder::HavingCondition);
369  if (!qb.exec()) {
370  inform("Failed to query overlapping parts, skipping test");
371  return;
372  }
373 
374  int count = 0;
375  while (qb.query().next()) {
376  ++count;
377  inform(QLatin1String("Found overlapping part data: ") + qb.query().value(0).toString());
378  // TODO: uh oh, this is bad, how do we recover from that?
379  }
380  qb.query().finish();
381 
382  if (count > 0) {
383  inform(QLatin1String("Found ") + QString::number(count) + QLatin1String(" overlapping parts - bad."));
384  }
385 }
386 
387 void StorageJanitor::verifyExternalParts()
388 {
389  QSet<QString> existingFiles;
390  QSet<QString> usedFiles;
391 
392  // list all files
393  const QString dataDir = StandardDirs::saveDir("data", QStringLiteral("file_db_data"));
395  while (it.hasNext()) {
396  existingFiles.insert(it.next());
397  }
398  existingFiles.remove(dataDir + QDir::separator() + QLatin1Char('.'));
399  existingFiles.remove(dataDir + QDir::separator() + QLatin1String(".."));
400  inform(QLatin1String("Found ") + QString::number(existingFiles.size()) + QLatin1String(" external files."));
401 
402  // list all parts from the db which claim to have an associated file
403  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
404  qb.addColumn(Part::dataColumn());
405  qb.addColumn(Part::pimItemIdColumn());
406  qb.addColumn(Part::idColumn());
407  qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
408  qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
409  if (!qb.exec()) {
410  inform("Failed to query existing parts, skipping test");
411  return;
412  }
413  while (qb.query().next()) {
414  const auto filename = qb.query().value(0).toByteArray();
415  const auto pimItemId = qb.query().value(1).value<Entity::Id>();
416  const auto partId = qb.query().value(2).value<Entity::Id>();
417  QString partPath;
418  if (!filename.isEmpty()) {
419  partPath = ExternalPartStorage::resolveAbsolutePath(filename);
420  } else {
421  partPath = ExternalPartStorage::resolveAbsolutePath(ExternalPartStorage::nameForPartId(partId));
422  }
423  if (existingFiles.contains(partPath)) {
424  usedFiles.insert(partPath);
425  } else {
426  inform(QLatin1String("Cleaning up missing external file: ") + partPath + QLatin1String(" for item: ") + QString::number(pimItemId)
427  + QLatin1String(" on part: ") + QString::number(partId));
428 
429  Part part;
430  part.setId(partId);
431  part.setPimItemId(pimItemId);
432  part.setData(QByteArray());
433  part.setDatasize(0);
434  part.setStorage(Part::Internal);
435  part.update();
436  }
437  }
438  qb.query().finish();
439  inform(QLatin1String("Found ") + QString::number(usedFiles.size()) + QLatin1String(" external parts."));
440 
441  // see what's left and move it to lost+found
442  const QSet<QString> unreferencedFiles = existingFiles - usedFiles;
443  if (!unreferencedFiles.isEmpty()) {
444  const QString lfDir = StandardDirs::saveDir("data", QStringLiteral("file_lost+found"));
445  for (const QString &file : unreferencedFiles) {
446  inform(QLatin1String("Found unreferenced external file: ") + file);
447  const QFileInfo f(file);
448  QFile::rename(file, lfDir + QDir::separator() + f.fileName());
449  }
450  inform(QStringLiteral("Moved %1 unreferenced files to lost+found.").arg(unreferencedFiles.size()));
451  } else {
452  inform("Found no unreferenced external files.");
453  }
454 }
455 
456 void StorageJanitor::findDirtyObjects()
457 {
459  cqb.setSubQueryMode(Query::Or);
460  cqb.addValueCondition(Collection::remoteIdColumn(), Query::Is, QVariant());
461  cqb.addValueCondition(Collection::remoteIdColumn(), Query::Equals, QString());
462  if (!cqb.exec()) {
463  inform("Failed to query collections without RID, skipping test");
464  return;
465  }
466  const Collection::List ridLessCols = cqb.result();
467  for (const Collection &col : ridLessCols) {
468  inform(QLatin1String("Collection \"") + col.name() + QLatin1String("\" (id: ") + QString::number(col.id()) + QLatin1String(") has no RID."));
469  }
470  inform(QLatin1String("Found ") + QString::number(ridLessCols.size()) + QLatin1String(" collections without RID."));
471 
473  iqb1.setSubQueryMode(Query::Or);
474  iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Is, QVariant());
475  iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, QString());
476  if (!iqb1.exec()) {
477  inform("Failed to query items without RID, skipping test");
478  return;
479  }
480  const PimItem::List ridLessItems = iqb1.result();
481  for (const PimItem &item : ridLessItems) {
482  inform(QLatin1String("Item \"") + QString::number(item.id()) + QLatin1String("\" in collection \"") + QString::number(item.collectionId())
483  + QLatin1String("\" has no RID."));
484  }
485  inform(QLatin1String("Found ") + QString::number(ridLessItems.size()) + QLatin1String(" items without RID."));
486 
488  iqb2.addValueCondition(PimItem::dirtyColumn(), Query::Equals, true);
489  iqb2.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
490  iqb2.addSortColumn(PimItem::idFullColumnName());
491  if (!iqb2.exec()) {
492  inform("Failed to query dirty items, skipping test");
493  return;
494  }
495  const PimItem::List dirtyItems = iqb2.result();
496  for (const PimItem &item : dirtyItems) {
497  inform(QLatin1String("Item \"") + QString::number(item.id()) + QLatin1String("\" has RID and is dirty."));
498  }
499  inform(QLatin1String("Found ") + QString::number(dirtyItems.size()) + QLatin1String(" dirty items."));
500 }
501 
502 void StorageJanitor::findRIDDuplicates()
503 {
504  QueryBuilder qb(Collection::tableName(), QueryBuilder::Select);
505  qb.addColumn(Collection::idColumn());
506  qb.addColumn(Collection::nameColumn());
507  qb.exec();
508 
509  while (qb.query().next()) {
510  const auto colId = qb.query().value(0).value<Collection::Id>();
511  const QString name = qb.query().value(1).toString();
512  inform(QStringLiteral("Checking ") + name);
513 
514  QueryBuilder duplicates(PimItem::tableName(), QueryBuilder::Select);
515  duplicates.addColumn(PimItem::remoteIdColumn());
516  duplicates.addColumn(QStringLiteral("count(") + PimItem::idColumn() + QStringLiteral(") as cnt"));
517  duplicates.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
518  duplicates.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
519  duplicates.addGroupColumn(PimItem::remoteIdColumn());
520  duplicates.addValueCondition(QStringLiteral("count(") + PimItem::idColumn() + QLatin1Char(')'), Query::Greater, 1, QueryBuilder::HavingCondition);
521  duplicates.exec();
522 
523  Akonadi::Server::Collection col = Akonadi::Server::Collection::retrieveById(colId);
524  const QVector<Akonadi::Server::MimeType> contentMimeTypes = col.mimeTypes();
525  QVariantList contentMimeTypesVariantList;
526  contentMimeTypesVariantList.reserve(contentMimeTypes.count());
527  for (const Akonadi::Server::MimeType &mimeType : contentMimeTypes) {
528  contentMimeTypesVariantList << mimeType.id();
529  }
530  while (duplicates.query().next()) {
531  const QString rid = duplicates.query().value(0).toString();
532 
533  Query::Condition condition(Query::And);
534  condition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, rid);
535  condition.addValueCondition(PimItem::mimeTypeIdColumn(), Query::NotIn, contentMimeTypesVariantList);
536  condition.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
537 
538  QueryBuilder items(PimItem::tableName(), QueryBuilder::Select);
539  items.addColumn(PimItem::idColumn());
540  items.addCondition(condition);
541  if (!items.exec()) {
542  inform(QStringLiteral("Error while deleting duplicates: ") + items.query().lastError().text());
543  continue;
544  }
545  QVariantList itemsIds;
546  while (items.query().next()) {
547  itemsIds.push_back(items.query().value(0));
548  }
549  items.query().finish();
550  if (itemsIds.isEmpty()) {
551  // the mimetype filter may have dropped some entries from the
552  // duplicates query
553  continue;
554  }
555 
556  inform(QStringLiteral("Found duplicates ") + rid);
557 
559  parts.addValueCondition(Part::pimItemIdFullColumnName(), Query::In, QVariant::fromValue(itemsIds));
560  parts.addValueCondition(Part::storageFullColumnName(), Query::Equals, static_cast<int>(Part::External));
561  if (parts.exec()) {
562  const auto partsList = parts.result();
563  for (const auto &part : partsList) {
564  bool exists = false;
565  const auto filename = ExternalPartStorage::resolveAbsolutePath(part.data(), &exists);
566  if (exists) {
567  QFile::remove(filename);
568  }
569  }
570  }
571 
572  items = QueryBuilder(PimItem::tableName(), QueryBuilder::Delete);
573  items.addCondition(condition);
574  if (!items.exec()) {
575  inform(QStringLiteral("Error while deleting duplicates ") + items.query().lastError().text());
576  }
577  }
578  duplicates.query().finish();
579  }
580  qb.query().finish();
581 }
582 
583 void StorageJanitor::vacuum()
584 {
585  const DbType::Type dbType = DbType::type(DataStore::self()->database());
586  if (dbType == DbType::MySQL || dbType == DbType::PostgreSQL) {
587  inform("vacuuming database, that'll take some time and require a lot of temporary disk space...");
588  const auto tables = allDatabaseTables();
589  for (const QString &table : tables) {
590  inform(QStringLiteral("optimizing table %1...").arg(table));
591 
592  QString queryStr;
593  if (dbType == DbType::MySQL) {
594  queryStr = QLatin1String("OPTIMIZE TABLE ") + table;
595  } else if (dbType == DbType::PostgreSQL) {
596  queryStr = QLatin1String("VACUUM FULL ANALYZE ") + table;
597  } else {
598  continue;
599  }
600  QSqlQuery q(DataStore::self()->database());
601  if (!q.exec(queryStr)) {
602  qCCritical(AKONADISERVER_LOG) << "failed to optimize table" << table << ":" << q.lastError().text();
603  }
604  }
605  inform("vacuum done");
606  } else {
607  inform("Vacuum not supported for this database backend.");
608  }
609 
610  Q_EMIT done();
611 }
612 
613 void StorageJanitor::checkSizeTreshold()
614 {
615  {
616  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
617  qb.addColumn(Part::idFullColumnName());
618  qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::Internal);
619  qb.addValueCondition(Part::datasizeFullColumnName(), Query::Greater, DbConfig::configuredDatabase()->sizeThreshold());
620  if (!qb.exec()) {
621  inform("Failed to query parts larger than threshold, skipping test");
622  return;
623  }
624 
625  QSqlQuery query = qb.query();
626  inform(QStringLiteral("Found %1 parts to be moved to external files").arg(query.size()));
627 
628  while (query.next()) {
629  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD"));
630  Part part = Part::retrieveById(query.value(0).toLongLong());
631  const QByteArray name = ExternalPartStorage::nameForPartId(part.id());
632  const QString partPath = ExternalPartStorage::resolveAbsolutePath(name);
633  QFile f(partPath);
634  if (f.exists()) {
635  qCDebug(AKONADISERVER_LOG) << "External payload file" << name << "already exists";
636  // That however is not a critical issue, since the part is not external,
637  // so we can safely overwrite it
638  }
639  if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
640  qCCritical(AKONADISERVER_LOG) << "Failed to open file" << name << "for writing";
641  continue;
642  }
643  if (f.write(part.data()) != part.datasize()) {
644  qCCritical(AKONADISERVER_LOG) << "Failed to write data to payload file" << name;
645  f.remove();
646  continue;
647  }
648 
649  part.setData(name);
650  part.setStorage(Part::External);
651  if (!part.update() || !transaction.commit()) {
652  qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
653  f.remove();
654  continue;
655  }
656 
657  inform(QStringLiteral("Moved part %1 from database into external file %2").arg(part.id()).arg(QString::fromLatin1(name)));
658  }
659  query.finish();
660  }
661 
662  {
663  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
664  qb.addColumn(Part::idFullColumnName());
665  qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
666  qb.addValueCondition(Part::datasizeFullColumnName(), Query::Less, DbConfig::configuredDatabase()->sizeThreshold());
667  if (!qb.exec()) {
668  inform("Failed to query parts smaller than threshold, skipping test");
669  return;
670  }
671 
672  QSqlQuery query = qb.query();
673  inform(QStringLiteral("Found %1 parts to be moved to database").arg(query.size()));
674 
675  while (query.next()) {
676  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD 2"));
677  Part part = Part::retrieveById(query.value(0).toLongLong());
678  const QString partPath = ExternalPartStorage::resolveAbsolutePath(part.data());
679  QFile f(partPath);
680  if (!f.exists()) {
681  qCCritical(AKONADISERVER_LOG) << "Part file" << part.data() << "does not exist";
682  continue;
683  }
684  if (!f.open(QIODevice::ReadOnly)) {
685  qCCritical(AKONADISERVER_LOG) << "Failed to open part file" << part.data() << "for reading";
686  continue;
687  }
688 
689  part.setStorage(Part::Internal);
690  part.setData(f.readAll());
691  if (part.data().size() != part.datasize()) {
692  qCCritical(AKONADISERVER_LOG) << "Sizes of" << part.id() << "data don't match";
693  continue;
694  }
695  if (!part.update() || !transaction.commit()) {
696  qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
697  continue;
698  }
699 
700  f.close();
701  f.remove();
702  inform(QStringLiteral("Moved part %1 from external file into database").arg(part.id()));
703  }
704  query.finish();
705  }
706 }
707 
708 void StorageJanitor::migrateToLevelledCacheHierarchy()
709 {
710  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
711  qb.addColumn(Part::idColumn());
712  qb.addColumn(Part::dataColumn());
713  qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
714  if (!qb.exec()) {
715  inform("Failed to query external payload parts, skipping test");
716  return;
717  }
718 
719  QSqlQuery query = qb.query();
720  while (query.next()) {
721  const qint64 id = query.value(0).toLongLong();
722  const QByteArray data = query.value(1).toByteArray();
723  const QString fileName = QString::fromUtf8(data);
724  bool oldExists = false;
725  bool newExists = false;
726  // Resolve the current path
727  const QString currentPath = ExternalPartStorage::resolveAbsolutePath(fileName, &oldExists);
728  // Resolve the new path with legacy fallback disabled, so that it always
729  // returns the new levelled-cache path, even when the old one exists
730  const QString newPath = ExternalPartStorage::resolveAbsolutePath(fileName, &newExists, false);
731  if (!oldExists) {
732  qCCritical(AKONADISERVER_LOG) << "Old payload part does not exist, skipping part" << fileName;
733  continue;
734  }
735  if (currentPath != newPath) {
736  if (newExists) {
737  qCCritical(AKONADISERVER_LOG) << "Part is in legacy location, but the destination file already exists, skipping part" << fileName;
738  continue;
739  }
740 
741  QFile f(currentPath);
742  if (!f.rename(newPath)) {
743  qCCritical(AKONADISERVER_LOG) << "Failed to move part from" << currentPath << " to " << newPath << ":" << f.errorString();
744  continue;
745  }
746  inform(QStringLiteral("Migrated part %1 to new levelled cache").arg(id));
747  }
748  }
749  query.finish();
750 }
751 
752 void StorageJanitor::findOrphanSearchIndexEntries()
753 {
754  QueryBuilder qb(Collection::tableName(), QueryBuilder::Select);
755  qb.addSortColumn(Collection::idColumn(), Query::Ascending);
756  qb.addColumn(Collection::idColumn());
757  qb.addColumn(Collection::isVirtualColumn());
758  if (!qb.exec()) {
759  inform("Failed to query collections, skipping test");
760  return;
761  }
762 
763  QDBusInterface iface(DBus::agentServiceName(QStringLiteral("akonadi_indexing_agent"), DBus::Agent),
764  QStringLiteral("/"),
765  QStringLiteral("org.freedesktop.Akonadi.Indexer"),
767  if (!iface.isValid()) {
768  inform("Akonadi Indexing Agent is not running, skipping test");
769  return;
770  }
771 
772  QSqlQuery query = qb.query();
773  while (query.next()) {
774  const qint64 colId = query.value(0).toLongLong();
775  // Skip virtual collections, they are not indexed
776  if (query.value(1).toBool()) {
777  inform(QStringLiteral("Skipping virtual Collection %1").arg(colId));
778  continue;
779  }
780 
781  inform(QStringLiteral("Checking Collection %1 search index...").arg(colId));
782  SearchRequest req("StorageJanitor", m_akonadi.searchManager(), m_akonadi.agentSearchManager());
783  req.setStoreResults(true);
784  req.setCollections({colId});
785  req.setRemoteSearch(false);
786  req.setQuery(QStringLiteral("{ }")); // empty query to match all
787  QStringList mts;
788  Collection col;
789  col.setId(colId);
790  const auto colMts = col.mimeTypes();
791  if (colMts.isEmpty()) {
792  // No mimetypes means we don't know which search store to look into,
793  // skip it.
794  continue;
795  }
796  mts.reserve(colMts.count());
797  for (const auto &mt : colMts) {
798  mts << mt.name();
799  }
800  req.setMimeTypes(mts);
801  req.exec();
802  auto searchResults = req.results();
803 
804  QueryBuilder iqb(PimItem::tableName(), QueryBuilder::Select);
805  iqb.addColumn(PimItem::idColumn());
806  iqb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
807  if (!iqb.exec()) {
808  inform(QStringLiteral("Failed to query items in collection %1").arg(colId));
809  continue;
810  }
811 
812  QSqlQuery itemQuery = iqb.query();
813  while (itemQuery.next()) {
814  searchResults.remove(itemQuery.value(0).toLongLong());
815  }
816  itemQuery.finish();
817 
818  if (!searchResults.isEmpty()) {
819  inform(QStringLiteral("Collection %1 search index contains %2 orphan items. Scheduling reindexing").arg(colId).arg(searchResults.count()));
820  iface.call(QDBus::NoBlock, QStringLiteral("reindexCollection"), colId);
821  }
822  }
823  query.finish();
824 }
825 
826 void StorageJanitor::ensureSearchCollection()
827 {
828  static const auto searchResourceName = QStringLiteral("akonadi_search_resource");
829 
830  auto searchResource = Resource::retrieveByName(searchResourceName);
831  if (!searchResource.isValid()) {
832  searchResource.setName(searchResourceName);
833  searchResource.setIsVirtual(true);
834  if (!searchResource.insert()) {
835  inform(QStringLiteral("Failed to create Search resource."));
836  return;
837  }
838  }
839 
840  auto searchCols = Collection::retrieveFiltered(Collection::resourceIdColumn(), searchResource.id());
841  if (searchCols.isEmpty()) {
842  Collection searchCol;
843  searchCol.setId(1);
844  searchCol.setName(QStringLiteral("Search"));
845  searchCol.setResource(searchResource);
846  searchCol.setIndexPref(Collection::False);
847  searchCol.setIsVirtual(true);
848  if (!searchCol.insert()) {
849  inform(QStringLiteral("Failed to create Search Collection"));
850  return;
851  }
852  }
853 }
854 
855 void StorageJanitor::inform(const char *msg)
856 {
857  inform(QLatin1String(msg));
858 }
859 
860 void StorageJanitor::inform(const QString &msg)
861 {
862  qCDebug(AKONADISERVER_LOG) << msg;
863  Q_EMIT information(msg);
864 }
void append(const T &value)
QVariant value(int index) const const
bool isEmpty() const const
Helper class for DataStore transaction handling.
Definition: transaction.h:22
void addGroupColumn(const QString &column)
Add a GROUP BY column.
bool remove(const T &value)
QString number(int n, int base)
void setId(Id identifier)
Sets the unique identifier of the collection.
Definition: collection.cpp:91
QString fromUtf8(const char *str, int size)
QVariant fromValue(const T &value)
bool remove()
QVector::iterator begin()
QDateTime currentDateTime()
T value() const const
QChar separator()
bool rename(const QString &newName)
void append(const T &value)
Represents a collection of PIM items.
Definition: collection.h:61
KCALUTILS_EXPORT QString mimeType()
bool registerObject(const QString &path, QObject *object, QDBusConnection::RegisterOptions options)
QByteArray toByteArray() const const
qlonglong toLongLong(bool *ok) const const
T & first()
void setColumnValue(const QString &column, const QVariant &value)
Sets a column to the given value (only valid for INSERT and UPDATE queries).
int size() const const
bool exec(const QString &query)
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
void reserve(int alloc)
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
int size() const const
QDBusConnection sessionBus()
bool registerService(const QString &serviceName)
QString name() const
bool unregisterService(const QString &serviceName)
Type
Supported database types.
Definition: dbtype.h:19
char * toString(const T &value)
QByteArray toUtf8() const const
QVector< T > result()
Returns the result of this SELECT query.
void information(QWidget *parent, const QString &text, const QString &caption=QString(), const QString &dontShowAgainName=QString(), Options options=Notify)
void finish()
bool next()
void disconnectFromBus(const QString &name)
void addColumn(const QString &col)
Adds the given column to a select query.
bool isEmpty() const const
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
QString join(const QString &separator) const const
void reserve(int size)
bool contains(const T &value) const const
QString & remove(int position, int n)
bool exec()
Executes the query, returns true on success.
QVector::iterator end()
QSqlError lastError() const const
QString text() const const
void unregisterObject(const QString &path, QDBusConnection::UnregisterMode mode)
QString name() const const
Helper class for creating and executing database SELECT queries.
void setName(QString value)
QString fromLatin1(const char *str, int size)
QString name(StandardShortcut id)
QSet::iterator insert(const T &value)
int count(const T &value) const const
void setSubQueryMode(Query::LogicOperator op, ConditionType type=WhereCondition)
Define how WHERE or HAVING conditions are combined.
int size() const const
Represents a WHERE condition tree.
Definition: query.h:61
QSqlQuery & query()
Returns the query, only valid after exec().
void setResource(const QString &identifier)
Sets the identifier of the resource owning the collection.
Definition: collection.cpp:307
void addSortColumn(const QString &column, Query::SortOrder order=Query::Ascending)
Add sort column.
qint64 Id
Describes the unique id type.
Definition: collection.h:79
void setName(const QString &name)
Sets the i18n'ed name of the collection.
Definition: collection.cpp:221
T value(int i) const const
bool isEmpty() const const
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:31
QString toString() const const
Helper integration between Akonadi and Qt.
This file is part of the KDE documentation.
Documentation copyright © 1996-2022 The KDE developers.
Generated on Mon Jun 27 2022 04:01:07 by doxygen 1.8.17 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.