Akonadi

storagejanitor.cpp
1 /*
2  Copyright (c) 2011 Volker Krause <[email protected]>
3 
4  This library is free software; you can redistribute it and/or modify it
5  under the terms of the GNU Library General Public License as published by
6  the Free Software Foundation; either version 2 of the License, or (at your
7  option) any later version.
8 
9  This library is distributed in the hope that it will be useful, but WITHOUT
10  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
12  License for more details.
13 
14  You should have received a copy of the GNU Library General Public License
15  along with this library; see the file COPYING.LIB. If not, write to the
16  Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  02110-1301, USA.
18 */
19 
20 #include "storagejanitor.h"
21 #include "akonadi.h"
22 #include "storage/queryhelper.h"
23 #include "storage/transaction.h"
24 #include "storage/datastore.h"
25 #include "storage/selectquerybuilder.h"
26 #include "storage/parthelper.h"
27 #include "storage/dbconfig.h"
28 #include "storage/collectionstatistics.h"
29 #include "search/searchrequest.h"
30 #include "search/searchmanager.h"
31 #include "resourcemanager.h"
32 #include "entities.h"
33 #include "agentmanagerinterface.h"
34 #include "akonadiserver_debug.h"
35 
36 #include <private/dbus_p.h>
37 #include <private/imapset_p.h>
38 #include <private/protocol_p.h>
39 #include <private/standarddirs_p.h>
40 #include <private/externalpartstorage_p.h>
41 
42 #include <QStringBuilder>
43 #include <QDBusConnection>
44 #include <QSqlQuery>
45 #include <QSqlError>
46 #include <QDir>
47 #include <QDirIterator>
48 #include <QDateTime>
49 
50 #include <algorithm>
51 
52 using namespace Akonadi;
53 using namespace Akonadi::Server;
54 
55 StorageJanitor::StorageJanitor(AkonadiServer &akonadi)
56  : AkThread(QStringLiteral("StorageJanitor"), QThread::IdlePriority)
57  , m_lostFoundCollectionId(-1)
58  , m_akonadi(akonadi)
59 {
60 }
61 
62 StorageJanitor::~StorageJanitor()
63 {
64  quitThread();
65 }
66 
67 void StorageJanitor::init()
68 {
69  AkThread::init();
70 
72  conn.registerService(DBus::serviceName(DBus::StorageJanitor));
73  conn.registerObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), this,
74  QDBusConnection::ExportScriptableSlots | QDBusConnection::ExportScriptableSignals);
75 }
76 
77 void StorageJanitor::quit()
78 {
80  conn.unregisterObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), QDBusConnection::UnregisterTree);
81  conn.unregisterService(DBus::serviceName(DBus::StorageJanitor));
82  conn.disconnectFromBus(conn.name());
83 
84  // Make sure all children are deleted within context of this thread
85  qDeleteAll(children());
86 
87  AkThread::quit();
88 }
89 
90 void StorageJanitor::check() // implementation of `akonadictl fsck`
91 {
92  m_lostFoundCollectionId = -1; // start with a fresh one each time
93 
94  inform("Looking for resources in the DB not matching a configured resource...");
95  findOrphanedResources();
96 
97  inform("Looking for collections not belonging to a valid resource...");
98  findOrphanedCollections();
99 
100  inform("Checking collection tree consistency...");
101  const Collection::List cols = Collection::retrieveAll();
102  std::for_each(cols.begin(), cols.end(), [this](const Collection & col) {
103  checkPathToRoot(col);
104  });
105 
106  inform("Looking for items not belonging to a valid collection...");
107  findOrphanedItems();
108 
109  inform("Looking for item parts not belonging to a valid item...");
110  findOrphanedParts();
111 
112  inform("Looking for item flags not belonging to a valid item...");
113  findOrphanedPimItemFlags();
114 
115  inform("Looking for overlapping external parts...");
116  findOverlappingParts();
117 
118  inform("Verifying external parts...");
119  verifyExternalParts();
120 
121  inform("Checking size treshold changes...");
122  checkSizeTreshold();
123 
124  inform("Looking for dirty objects...");
125  findDirtyObjects();
126 
127  inform("Looking for rid-duplicates not matching the content mime-type of the parent collection");
128  findRIDDuplicates();
129 
130  inform("Migrating parts to new cache hierarchy...");
131  migrateToLevelledCacheHierarchy();
132 
133  inform("Checking search index consistency...");
134  findOrphanSearchIndexEntries();
135 
136  inform("Flushing collection statistics memory cache...");
137  m_akonadi.collectionStatistics().expireCache();
138 
139  inform("Making sure virtual search resource and collections exist");
140  ensureSearchCollection();
141 
142  /* TODO some ideas for further checks:
143  * the collection tree is non-cyclic
144  * content type constraints of collections are not violated
145  * find unused flags
146  * find unused mimetypes
147  * check for dead entries in relation tables
148  * check if part size matches file size
149  */
150 
151  inform("Consistency check done.");
152 
153  Q_EMIT done();
154 }
155 
156 qint64 StorageJanitor::lostAndFoundCollection()
157 {
158  if (m_lostFoundCollectionId > 0) {
159  return m_lostFoundCollectionId;
160  }
161 
162  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR LOST+FOUND"));
163  Resource lfRes = Resource::retrieveByName(QStringLiteral("akonadi_lost+found_resource"));
164  if (!lfRes.isValid()) {
165  lfRes.setName(QStringLiteral("akonadi_lost+found_resource"));
166  if (!lfRes.insert()) {
167  qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found resource!";
168  }
169  }
170 
171  Collection lfRoot;
173  qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, lfRes.id());
174  qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant());
175  if (!qb.exec()) {
176  qCCritical(AKONADISERVER_LOG) << "Failed to query top level collections";
177  return -1;
178  }
179  const Collection::List cols = qb.result();
180  if (cols.size() > 1) {
181  qCCritical(AKONADISERVER_LOG) << "More than one top-level lost+found collection!?";
182  } else if (cols.size() == 1) {
183  lfRoot = cols.first();
184  } else {
185  lfRoot.setName(QStringLiteral("lost+found"));
186  lfRoot.setResourceId(lfRes.id());
187  lfRoot.setCachePolicyLocalParts(QStringLiteral("ALL"));
188  lfRoot.setCachePolicyCacheTimeout(-1);
189  lfRoot.setCachePolicyInherit(false);
190  if (!lfRoot.insert()) {
191  qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found root.";
192  }
193  DataStore::self()->notificationCollector()->collectionAdded(lfRoot, lfRes.name().toUtf8());
194  }
195 
196  Collection lfCol;
197  lfCol.setName(QDateTime::currentDateTime().toString(QStringLiteral("yyyy-MM-dd hh:mm:ss")));
198  lfCol.setResourceId(lfRes.id());
199  lfCol.setParentId(lfRoot.id());
200  if (!lfCol.insert()) {
201  qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found collection!";
202  }
203 
204  Q_FOREACH (const MimeType &mt, MimeType::retrieveAll()) {
205  lfCol.addMimeType(mt);
206  }
207 
208  DataStore::self()->notificationCollector()->collectionAdded(lfCol, lfRes.name().toUtf8());
209 
210  transaction.commit();
211  m_lostFoundCollectionId = lfCol.id();
212  return m_lostFoundCollectionId;
213 }
214 
215 void StorageJanitor::findOrphanedResources()
216 {
218  OrgFreedesktopAkonadiAgentManagerInterface iface(
219  DBus::serviceName(DBus::Control),
220  QStringLiteral("/AgentManager"),
222  this);
223  if (!iface.isValid()) {
224  inform(QStringLiteral("ERROR: Couldn't talk to %1").arg(DBus::Control));
225  return;
226  }
227  const QStringList knownResources = iface.agentInstances();
228  if (knownResources.isEmpty()) {
229  inform(QStringLiteral("ERROR: no known resources. This must be a mistake?"));
230  return;
231  }
232  qbres.addValueCondition(Resource::nameFullColumnName(), Query::NotIn, QVariant(knownResources));
233  qbres.addValueCondition(Resource::idFullColumnName(), Query::NotEquals, 1); // skip akonadi_search_resource
234  if (!qbres.exec()) {
235  inform("Failed to query known resources, skipping test");
236  return;
237  }
238  //qCDebug(AKONADISERVER_LOG) << "SQL:" << qbres.query().lastQuery();
239  const Resource::List orphanResources = qbres.result();
240  const int orphanResourcesSize(orphanResources.size());
241  if (orphanResourcesSize > 0) {
242  QStringList resourceNames;
243  resourceNames.reserve(orphanResourcesSize);
244  for (const Resource &resource : orphanResources) {
245  resourceNames.append(resource.name());
246  }
247  inform(QStringLiteral("Found %1 orphan resources: %2").arg(orphanResourcesSize). arg(resourceNames.join(QLatin1Char(','))));
248  for (const QString &resourceName : qAsConst(resourceNames)) {
249  inform(QStringLiteral("Removing resource %1").arg(resourceName));
250  m_akonadi.resourceManager().removeResourceInstance(resourceName);
251  }
252  }
253 }
254 
255 void StorageJanitor::findOrphanedCollections()
256 {
258  qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
259  qb.addValueCondition(Resource::idFullColumnName(), Query::Is, QVariant());
260 
261  if (!qb.exec()) {
262  inform("Failed to query orphaned collections, skipping test");
263  return;
264  }
265  const Collection::List orphans = qb.result();
266  if (!orphans.isEmpty()) {
267  inform(QLatin1String("Found ") + QString::number(orphans.size()) + QLatin1String(" orphan collections."));
268  // TODO: attach to lost+found resource
269  }
270 }
271 
272 void StorageJanitor::checkPathToRoot(const Collection &col)
273 {
274  if (col.parentId() == 0) {
275  return;
276  }
277  const Collection parent = col.parent();
278  if (!parent.isValid()) {
279  inform(QLatin1String("Collection \"") + col.name() + QLatin1String("\" (id: ") + QString::number(col.id())
280  + QLatin1String(") has no valid parent."));
281  // TODO fix that by attaching to a top-level lost+found folder
282  return;
283  }
284 
285  if (col.resourceId() != parent.resourceId()) {
286  inform(QLatin1String("Collection \"") + col.name() + QLatin1String("\" (id: ") + QString::number(col.id())
287  + QLatin1String(") belongs to a different resource than its parent."));
288  // can/should we actually fix that?
289  }
290 
291  checkPathToRoot(parent);
292 }
293 
294 void StorageJanitor::findOrphanedItems()
295 {
297  qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
298  qb.addValueCondition(Collection::idFullColumnName(), Query::Is, QVariant());
299  if (!qb.exec()) {
300  inform("Failed to query orphaned items, skipping test");
301  return;
302  }
303  const PimItem::List orphans = qb.result();
304  if (!orphans.isEmpty()) {
305  inform(QLatin1String("Found ") + QString::number(orphans.size()) + QLatin1String(" orphan items."));
306  // Attach to lost+found collection
307  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR ORPHANS"));
308  QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
309  qint64 col = lostAndFoundCollection();
310  if (col == -1) {
311  return;
312  }
313  qb.setColumnValue(PimItem::collectionIdColumn(), col);
314  QVector<ImapSet::Id> imapIds;
315  imapIds.reserve(orphans.count());
316  for (const PimItem &item : qAsConst(orphans)) {
317  imapIds.append(item.id());
318  }
319  ImapSet set;
320  set.add(imapIds);
321  QueryHelper::setToQuery(set, PimItem::idFullColumnName(), qb);
322  if (qb.exec() && transaction.commit()) {
323  inform(QLatin1String("Moved orphan items to collection ") + QString::number(col));
324  } else {
325  inform(QLatin1String("Error moving orphan items to collection ") + QString::number(col) + QLatin1String(" : ") + qb.query().lastError().text());
326  }
327  }
328 }
329 
330 void StorageJanitor::findOrphanedParts()
331 {
333  qb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
334  qb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
335  if (!qb.exec()) {
336  inform("Failed to query orphaned parts, skipping test");
337  return;
338  }
339  const Part::List orphans = qb.result();
340  if (!orphans.isEmpty()) {
341  inform(QLatin1String("Found ") + QString::number(orphans.size()) + QLatin1String(" orphan parts."));
342  // TODO: create lost+found items for those? delete?
343  }
344 }
345 
346 void StorageJanitor:: findOrphanedPimItemFlags()
347 {
348  QueryBuilder sqb(PimItemFlagRelation::tableName(), QueryBuilder::Select);
349  sqb.addColumn(PimItemFlagRelation::leftFullColumnName());
350  sqb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), PimItemFlagRelation::leftFullColumnName(), PimItem::idFullColumnName());
351  sqb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
352  if (!sqb.exec()) {
353  inform("Failed to query orphaned item flags, skipping test");
354  return;
355  }
356  QVector<ImapSet::Id> imapIds;
357  int count = 0;
358  while (sqb.query().next()) {
359  ++count;
360  imapIds.append(sqb.query().value(0).toInt());
361  }
362  sqb.query().finish();
363  if (count > 0) {
364  ImapSet set;
365  set.add(imapIds);
366  QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete);
367  QueryHelper::setToQuery(set, PimItemFlagRelation::leftFullColumnName(), qb);
368  if (!qb.exec()) {
369  qCCritical(AKONADISERVER_LOG) << "Error:" << qb.query().lastError().text();
370  return;
371  }
372 
373  inform(QLatin1String("Found and deleted ") + QString::number(count) + QLatin1String(" orphan pim item flags."));
374  }
375 }
376 
377 void StorageJanitor::findOverlappingParts()
378 {
379  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
380  qb.addColumn(Part::dataColumn());
381  qb.addColumn(QLatin1String("count(") + Part::idColumn() + QLatin1String(") as cnt"));
382  qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
383  qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
384  qb.addGroupColumn(Part::dataColumn());
385  qb.addValueCondition(QLatin1String("count(") + Part::idColumn() + QLatin1String(")"), Query::Greater, 1, QueryBuilder::HavingCondition);
386  if (!qb.exec()) {
387  inform("Failed to query overlapping parts, skipping test");
388  return;
389  }
390 
391  int count = 0;
392  while (qb.query().next()) {
393  ++count;
394  inform(QLatin1String("Found overlapping part data: ") + qb.query().value(0).toString());
395  // TODO: uh oh, this is bad, how do we recover from that?
396  }
397  qb.query().finish();
398 
399  if (count > 0) {
400  inform(QLatin1String("Found ") + QString::number(count) + QLatin1String(" overlapping parts - bad."));
401  }
402 }
403 
404 void StorageJanitor::verifyExternalParts()
405 {
406  QSet<QString> existingFiles;
407  QSet<QString> usedFiles;
408 
409  // list all files
410  const QString dataDir = StandardDirs::saveDir("data", QStringLiteral("file_db_data"));
411  QDirIterator it(dataDir, QDir::Files, QDirIterator::Subdirectories);
412  while (it.hasNext()) {
413  existingFiles.insert(it.next());
414  }
415  existingFiles.remove(dataDir + QDir::separator() + QLatin1Char('.'));
416  existingFiles.remove(dataDir + QDir::separator() + QLatin1String(".."));
417  inform(QLatin1String("Found ") + QString::number(existingFiles.size()) + QLatin1String(" external files."));
418 
419  // list all parts from the db which claim to have an associated file
420  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
421  qb.addColumn(Part::dataColumn());
422  qb.addColumn(Part::pimItemIdColumn());
423  qb.addColumn(Part::idColumn());
424  qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
425  qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
426  if (!qb.exec()) {
427  inform("Failed to query existing parts, skipping test");
428  return;
429  }
430  while (qb.query().next()) {
431  const auto filename = qb.query().value(0).toByteArray();
432  const Entity::Id pimItemId = qb.query().value(1).value<Entity::Id>();
433  const Entity::Id partId = qb.query().value(2).value<Entity::Id>();
434  QString partPath;
435  if (!filename.isEmpty()) {
436  partPath = ExternalPartStorage::resolveAbsolutePath(filename);
437  } else {
438  partPath = ExternalPartStorage::resolveAbsolutePath(ExternalPartStorage::nameForPartId(partId));
439  }
440  if (existingFiles.contains(partPath)) {
441  usedFiles.insert(partPath);
442  } else {
443  inform(QLatin1String("Cleaning up missing external file: ") + partPath + QLatin1String(" for item: ") + QString::number(pimItemId) + QLatin1String(" on part: ") + QString::number(partId));
444 
445  Part part;
446  part.setId(partId);
447  part.setPimItemId(pimItemId);
448  part.setData(QByteArray());
449  part.setDatasize(0);
450  part.setStorage(Part::Internal);
451  part.update();
452  }
453  }
454  qb.query().finish();
455  inform(QLatin1String("Found ") + QString::number(usedFiles.size()) + QLatin1String(" external parts."));
456 
457  // see what's left and move it to lost+found
458  const QSet<QString> unreferencedFiles = existingFiles - usedFiles;
459  if (!unreferencedFiles.isEmpty()) {
460  const QString lfDir = StandardDirs::saveDir("data", QStringLiteral("file_lost+found"));
461  for (const QString &file : unreferencedFiles) {
462  inform(QLatin1String("Found unreferenced external file: ") + file);
463  const QFileInfo f(file);
464  QFile::rename(file, lfDir + QDir::separator() + f.fileName());
465  }
466  inform(QStringLiteral("Moved %1 unreferenced files to lost+found.").arg(unreferencedFiles.size()));
467  } else {
468  inform("Found no unreferenced external files.");
469  }
470 }
471 
472 void StorageJanitor::findDirtyObjects()
473 {
475  cqb.setSubQueryMode(Query::Or);
476  cqb.addValueCondition(Collection::remoteIdColumn(), Query::Is, QVariant());
477  cqb.addValueCondition(Collection::remoteIdColumn(), Query::Equals, QString());
478  if (!cqb.exec()) {
479  inform("Failed to query collections without RID, skipping test");
480  return;
481  }
482  const Collection::List ridLessCols = cqb.result();
483  for (const Collection &col : ridLessCols) {
484  inform(QLatin1String("Collection \"") + col.name() + QLatin1String("\" (id: ") + QString::number(col.id())
485  + QLatin1String(") has no RID."));
486  }
487  inform(QLatin1String("Found ") + QString::number(ridLessCols.size()) + QLatin1String(" collections without RID."));
488 
490  iqb1.setSubQueryMode(Query::Or);
491  iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Is, QVariant());
492  iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, QString());
493  if (!iqb1.exec()) {
494  inform("Failed to query items without RID, skipping test");
495  return;
496  }
497  const PimItem::List ridLessItems = iqb1.result();
498  for (const PimItem &item : ridLessItems) {
499  inform(QLatin1String("Item \"") + QString::number(item.id()) + QLatin1String("\" in collection \"") +
500  QString::number(item.collectionId()) + QLatin1String("\" has no RID."));
501  }
502  inform(QLatin1String("Found ") + QString::number(ridLessItems.size()) + QLatin1String(" items without RID."));
503 
505  iqb2.addValueCondition(PimItem::dirtyColumn(), Query::Equals, true);
506  iqb2.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
507  iqb2.addSortColumn(PimItem::idFullColumnName());
508  if (!iqb2.exec()) {
509  inform("Failed to query dirty items, skipping test");
510  return;
511  }
512  const PimItem::List dirtyItems = iqb2.result();
513  for (const PimItem &item : dirtyItems) {
514  inform(QLatin1String("Item \"") + QString::number(item.id()) + QLatin1String("\" has RID and is dirty."));
515  }
516  inform(QLatin1String("Found ") + QString::number(dirtyItems.size()) + QLatin1String(" dirty items."));
517 }
518 
519 void StorageJanitor::findRIDDuplicates()
520 {
521  QueryBuilder qb(Collection::tableName(), QueryBuilder::Select);
522  qb.addColumn(Collection::idColumn());
523  qb.addColumn(Collection::nameColumn());
524  qb.exec();
525 
526  while (qb.query().next()) {
527  const Collection::Id colId = qb.query().value(0).value<Collection::Id>();
528  const QString name = qb.query().value(1).toString();
529  inform(QStringLiteral("Checking ") + name);
530 
531  QueryBuilder duplicates(PimItem::tableName(), QueryBuilder::Select);
532  duplicates.addColumn(PimItem::remoteIdColumn());
533  duplicates.addColumn(QStringLiteral("count(") + PimItem::idColumn() + QStringLiteral(") as cnt"));
534  duplicates.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
535  duplicates.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
536  duplicates.addGroupColumn(PimItem::remoteIdColumn());
537  duplicates.addValueCondition(QStringLiteral("count(") + PimItem::idColumn() + QLatin1Char(')'), Query::Greater, 1, QueryBuilder::HavingCondition);
538  duplicates.exec();
539 
540  Akonadi::Server::Collection col = Akonadi::Server::Collection::retrieveById(colId);
541  const QVector<Akonadi::Server::MimeType> contentMimeTypes = col.mimeTypes();
542  QVariantList contentMimeTypesVariantList;
543  contentMimeTypesVariantList.reserve(contentMimeTypes.count());
544  for (const Akonadi::Server::MimeType &mimeType : contentMimeTypes) {
545  contentMimeTypesVariantList << mimeType.id();
546  }
547  while (duplicates.query().next()) {
548  const QString rid = duplicates.query().value(0).toString();
549 
550  Query::Condition condition(Query::And);
551  condition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, rid);
552  condition.addValueCondition(PimItem::mimeTypeIdColumn(), Query::NotIn, contentMimeTypesVariantList);
553  condition.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
554 
555  QueryBuilder items(PimItem::tableName(), QueryBuilder::Select);
556  items.addColumn(PimItem::idColumn());
557  items.addCondition(condition);
558  if (!items.exec()) {
559  inform(QStringLiteral("Error while deleting duplicates: ") + items.query().lastError().text());
560  continue;
561  }
562  QVariantList itemsIds;
563  while (items.query().next()) {
564  itemsIds.push_back(items.query().value(0));
565  }
566  items.query().finish();
567  if (itemsIds.isEmpty()) {
568  // the mimetype filter may have dropped some entries from the
569  // duplicates query
570  continue;
571  }
572 
573  inform(QStringLiteral("Found duplicates ") + rid);
574 
576  parts.addValueCondition(Part::pimItemIdFullColumnName(), Query::In, QVariant::fromValue(itemsIds));
577  parts.addValueCondition(Part::storageFullColumnName(), Query::Equals, (int) Part::External);
578  if (parts.exec()) {
579  const auto partsList = parts.result();
580  for (const auto &part : partsList) {
581  bool exists = false;
582  const auto filename = ExternalPartStorage::resolveAbsolutePath(part.data(), &exists);
583  if (exists) {
584  QFile::remove(filename);
585  }
586  }
587  }
588 
589  items = QueryBuilder(PimItem::tableName(), QueryBuilder::Delete);
590  items.addCondition(condition);
591  if (!items.exec()) {
592  inform(QStringLiteral("Error while deleting duplicates ") + items.query().lastError().text());
593  }
594  }
595  duplicates.query().finish();
596  }
597  qb.query().finish();
598 }
599 
600 void StorageJanitor::vacuum()
601 {
602  const DbType::Type dbType = DbType::type(DataStore::self()->database());
603  if (dbType == DbType::MySQL || dbType == DbType::PostgreSQL) {
604  inform("vacuuming database, that'll take some time and require a lot of temporary disk space...");
605  Q_FOREACH (const QString &table, allDatabaseTables()) {
606  inform(QStringLiteral("optimizing table %1...").arg(table));
607 
608  QString queryStr;
609  if (dbType == DbType::MySQL) {
610  queryStr = QLatin1String("OPTIMIZE TABLE ") + table;
611  } else if (dbType == DbType::PostgreSQL) {
612  queryStr = QLatin1String("VACUUM FULL ANALYZE ") + table;
613  } else {
614  continue;
615  }
616  QSqlQuery q(DataStore::self()->database());
617  if (!q.exec(queryStr)) {
618  qCCritical(AKONADISERVER_LOG) << "failed to optimize table" << table << ":" << q.lastError().text();
619  }
620  }
621  inform("vacuum done");
622  } else {
623  inform("Vacuum not supported for this database backend.");
624  }
625 
626  Q_EMIT done();
627 }
628 
629 void StorageJanitor::checkSizeTreshold()
630 {
631  {
632  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
633  qb.addColumn(Part::idFullColumnName());
634  qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::Internal);
635  qb.addValueCondition(Part::datasizeFullColumnName(), Query::Greater, DbConfig::configuredDatabase()->sizeThreshold());
636  if (!qb.exec()) {
637  inform("Failed to query parts larger than treshold, skipping test");
638  return;
639  }
640 
641  QSqlQuery query = qb.query();
642  inform(QStringLiteral("Found %1 parts to be moved to external files").arg(query.size()));
643 
644  while (query.next()) {
645  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD"));
646  Part part = Part::retrieveById(query.value(0).toLongLong());
647  const QByteArray name = ExternalPartStorage::nameForPartId(part.id());
648  const QString partPath = ExternalPartStorage::resolveAbsolutePath(name);
649  QFile f(partPath);
650  if (f.exists()) {
651  qCDebug(AKONADISERVER_LOG) << "External payload file" << name << "already exists";
652  // That however is not a critical issue, since the part is not external,
653  // so we can safely overwrite it
654  }
655  if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
656  qCCritical(AKONADISERVER_LOG) << "Failed to open file" << name << "for writing";
657  continue;
658  }
659  if (f.write(part.data()) != part.datasize()) {
660  qCCritical(AKONADISERVER_LOG) << "Failed to write data to payload file" << name;
661  f.remove();
662  continue;
663  }
664 
665  part.setData(name);
666  part.setStorage(Part::External);
667  if (!part.update() || !transaction.commit()) {
668  qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
669  f.remove();
670  continue;
671  }
672 
673  inform(QStringLiteral("Moved part %1 from database into external file %2").arg(part.id()).arg(QString::fromLatin1(name)));
674  }
675  query.finish();
676  }
677 
678  {
679  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
680  qb.addColumn(Part::idFullColumnName());
681  qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
682  qb.addValueCondition(Part::datasizeFullColumnName(), Query::Less, DbConfig::configuredDatabase()->sizeThreshold());
683  if (!qb.exec()) {
684  inform("Failed to query parts smaller than treshold, skipping test");
685  return;
686  }
687 
688  QSqlQuery query = qb.query();
689  inform(QStringLiteral("Found %1 parts to be moved to database").arg(query.size()));
690 
691  while (query.next()) {
692  Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD 2"));
693  Part part = Part::retrieveById(query.value(0).toLongLong());
694  const QString partPath = ExternalPartStorage::resolveAbsolutePath(part.data());
695  QFile f(partPath);
696  if (!f.exists()) {
697  qCCritical(AKONADISERVER_LOG) << "Part file" << part.data() << "does not exist";
698  continue;
699  }
700  if (!f.open(QIODevice::ReadOnly)) {
701  qCCritical(AKONADISERVER_LOG) << "Failed to open part file" << part.data() << "for reading";
702  continue;
703  }
704 
705  part.setStorage(Part::Internal);
706  part.setData(f.readAll());
707  if (part.data().size() != part.datasize()) {
708  qCCritical(AKONADISERVER_LOG) << "Sizes of" << part.id() << "data don't match";
709  continue;
710  }
711  if (!part.update() || !transaction.commit()) {
712  qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
713  continue;
714  }
715 
716  f.close();
717  f.remove();
718  inform(QStringLiteral("Moved part %1 from external file into database").arg(part.id()));
719  }
720  query.finish();
721  }
722 }
723 
724 void StorageJanitor::migrateToLevelledCacheHierarchy()
725 {
726  QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
727  qb.addColumn(Part::idColumn());
728  qb.addColumn(Part::dataColumn());
729  qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
730  if (!qb.exec()) {
731  inform("Failed to query external payload parts, skipping test");
732  return;
733  }
734 
735  QSqlQuery query = qb.query();
736  while (query.next()) {
737  const qint64 id = query.value(0).toLongLong();
738  const QByteArray data = query.value(1).toByteArray();
739  const QString fileName = QString::fromUtf8(data);
740  bool oldExists = false, newExists = false;
741  // Resolve the current path
742  const QString currentPath = ExternalPartStorage::resolveAbsolutePath(fileName, &oldExists);
743  // Resolve the new path with legacy fallback disabled, so that it always
744  // returns the new levelled-cache path, even when the old one exists
745  const QString newPath = ExternalPartStorage::resolveAbsolutePath(fileName, &newExists, false);
746  if (!oldExists) {
747  qCCritical(AKONADISERVER_LOG) << "Old payload part does not exist, skipping part" << fileName;
748  continue;
749  }
750  if (currentPath != newPath) {
751  if (newExists) {
752  qCCritical(AKONADISERVER_LOG) << "Part is in legacy location, but the destination file already exists, skipping part" << fileName;
753  continue;
754  }
755 
756  QFile f(currentPath);
757  if (!f.rename(newPath)) {
758  qCCritical(AKONADISERVER_LOG) << "Failed to move part from" << currentPath << " to " << newPath << ":" << f.errorString();
759  continue;
760  }
761  inform(QStringLiteral("Migrated part %1 to new levelled cache").arg(id));
762  }
763  }
764  query.finish();
765 }
766 
767 void StorageJanitor::findOrphanSearchIndexEntries()
768 {
769  QueryBuilder qb(Collection::tableName(), QueryBuilder::Select);
770  qb.addSortColumn(Collection::idColumn(), Query::Ascending);
771  qb.addColumn(Collection::idColumn());
772  qb.addColumn(Collection::isVirtualColumn());
773  if (!qb.exec()) {
774  inform("Failed to query collections, skipping test");
775  return;
776  }
777 
778  QDBusInterface iface(DBus::agentServiceName(QStringLiteral("akonadi_indexing_agent"), DBus::Agent),
779  QStringLiteral("/"),
780  QStringLiteral("org.freedesktop.Akonadi.Indexer"),
782  if (!iface.isValid()) {
783  inform("Akonadi Indexing Agent is not running, skipping test");
784  return;
785  }
786 
787  QSqlQuery query = qb.query();
788  while (query.next()) {
789  const qint64 colId = query.value(0).toLongLong();
790  // Skip virtual collections, they are not indexed
791  if (query.value(1).toBool()) {
792  inform(QStringLiteral("Skipping virtual Collection %1").arg(colId));
793  continue;
794  }
795 
796  inform(QStringLiteral("Checking Collection %1 search index...").arg(colId));
797  SearchRequest req("StorageJanitor", m_akonadi.searchManager(), m_akonadi.agentSearchManager());
798  req.setStoreResults(true);
799  req.setCollections({ colId });
800  req.setRemoteSearch(false);
801  req.setQuery(QStringLiteral("{ }")); // empty query to match all
802  QStringList mts;
803  Collection col;
804  col.setId(colId);
805  const auto colMts = col.mimeTypes();
806  if (colMts.isEmpty()) {
807  // No mimetypes means we don't know which search store to look into,
808  // skip it.
809  continue;
810  }
811  mts.reserve(colMts.count());
812  for (const auto &mt : colMts) {
813  mts << mt.name();
814  }
815  req.setMimeTypes(mts);
816  req.exec();
817  auto searchResults = req.results();
818 
819  QueryBuilder iqb(PimItem::tableName(), QueryBuilder::Select);
820  iqb.addColumn(PimItem::idColumn());
821  iqb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
822  if (!iqb.exec()) {
823  inform(QStringLiteral("Failed to query items in collection %1").arg(colId));
824  continue;
825  }
826 
827  QSqlQuery itemQuery = iqb.query();
828  while (itemQuery.next()) {
829  searchResults.remove(itemQuery.value(0).toLongLong());
830  }
831  itemQuery.finish();
832 
833  if (!searchResults.isEmpty()) {
834  inform(QStringLiteral("Collection %1 search index contains %2 orphan items. Scheduling reindexing").arg(colId).arg(searchResults.count()));
835  iface.call(QDBus::NoBlock, QStringLiteral("reindexCollection"), colId);
836  }
837  }
838  query.finish();
839 }
840 
841 void StorageJanitor::ensureSearchCollection()
842 {
843  static const auto searchResourceName = QStringLiteral("akonadi_search_resource");
844 
845  auto searchResource = Resource::retrieveByName(searchResourceName);
846  if (!searchResource.isValid()) {
847  searchResource.setName(searchResourceName);
848  searchResource.setIsVirtual(true);
849  if (!searchResource.insert()) {
850  inform(QStringLiteral("Failed to create Search resource."));
851  return;
852  }
853  }
854 
855  auto searchCols = Collection::retrieveFiltered(Collection::resourceIdColumn(), searchResource.id());
856  if (searchCols.isEmpty()) {
857  Collection searchCol;
858  searchCol.setId(1);
859  searchCol.setName(QStringLiteral("Search"));
860  searchCol.setResource(searchResource);
861  searchCol.setIndexPref(Collection::False);
862  searchCol.setIsVirtual(true);
863  if (!searchCol.insert()) {
864  inform(QStringLiteral("Failed to create Search Collection"));
865  return;
866  }
867  }
868 }
869 
870 
871 void StorageJanitor::inform(const char *msg)
872 {
873  inform(QLatin1String(msg));
874 }
875 
876 void StorageJanitor::inform(const QString &msg)
877 {
878  qCDebug(AKONADISERVER_LOG) << msg;
879  Q_EMIT information(msg);
880 }
QString next()
qlonglong toLongLong(bool *ok) const const
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
QByteArray toByteArray() const const
bool isValid() const const
int size() const const
bool isValid() const
Returns whether the collection is valid.
Definition: collection.cpp:137
QString name() const
Returns the i18n&#39;ed name of the collection.
Definition: collection.cpp:225
QSqlQuery & query()
Returns the query, only valid after exec().
void append(const T &value)
void setColumnValue(const QString &column, const QVariant &value)
Sets a column to the given value (only valid for INSERT and UPDATE queries).
QVector::iterator begin()
bool remove()
QString errorString() const const
Type
Supported database types.
Definition: dbtype.h:35
int size() const const
void reserve(int alloc)
bool exec(const QString &query)
Represents a collection of PIM items.
Definition: collection.h:76
qint64 Id
Describes the unique id type.
Definition: collection.h:82
bool rename(const QString &newName)
const QObjectList & children() const const
void setSubQueryMode(Query::LogicOperator op, ConditionType type=WhereCondition)
Define how WHERE or HAVING conditions are combined.
void setId(Id identifier)
Sets the unique identifier of the collection.
Definition: collection.cpp:107
bool registerObject(const QString &path, QObject *object, QDBusConnection::RegisterOptions options)
T value() const const
QDBusConnection sessionBus()
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
T & first()
QString join(const QString &separator) const const
bool exists() const const
QSet::iterator insert(const T &value)
QChar separator()
void setName(const QString &name)
Sets the i18n&#39;ed name of the collection.
Definition: collection.cpp:237
void addColumn(const QString &col)
Adds the given column to a select query.
QString number(int n, int base)
void append(const T &value)
QString fromUtf8(const char *str, int size)
Helper class for DataStore transaction handling.
Definition: transaction.h:37
int toInt(bool *ok) const const
QVariant value(int index) const const
void disconnectFromBus(const QString &name)
QString fileName() const const
void addGroupColumn(const QString &column)
Add a GROUP BY column.
void addSortColumn(const QString &column, Query::SortOrder order=Query::Ascending)
Add sort column.
bool isEmpty() const const
bool next()
QVector< T > result()
Returns the result of this SELECT query.
bool commit()
Commits the transaction.
Definition: transaction.cpp:42
bool unregisterService(const QString &serviceName)
void unregisterObject(const QString &path, QDBusConnection::UnregisterMode mode)
virtual bool open(QIODevice::OpenMode mode) override
void reserve(int size)
QString name() const const
QVariant fromValue(const T &value)
bool contains(const T &value) const const
Helper class for creating and executing database SELECT queries.
QDateTime currentDateTime()
Id id() const
Returns the unique identifier of the collection.
Definition: collection.cpp:112
bool remove(const T &value)
void finish()
bool isEmpty() const const
Helper integration between Akonadi and Qt.
int count(const T &value) const const
QSqlError lastError() const const
bool toBool() const const
qint64 write(const char *data, qint64 maxSize)
QString fromLatin1(const char *str, int size)
Represents a WHERE condition tree.
Definition: query.h:77
QString text() const const
bool hasNext() const const
QObject * parent() const const
int size() const const
QString toString() const const
QVector::iterator end()
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:45
void setResource(const QString &identifier)
Sets the identifier of the resource owning the collection.
Definition: collection.cpp:323
void addCondition(const Query::Condition &condition, ConditionType type=WhereCondition)
Add a WHERE condition.
Q_EMITQ_EMIT
bool registerService(const QString &serviceName)
bool exec()
Executes the query, returns true on success.
void addValueCondition(const QString &column, CompareOperator op, const QVariant &value)
Add a WHERE condition which compares a column with a given value.
Definition: query.cpp:25
QByteArray toUtf8() const const
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Fri Jun 5 2020 23:08:56 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.