Akonadi

storagejanitor.cpp
1/*
2 SPDX-FileCopyrightText: 2011 Volker Krause <vkrause@kde.org>
3
4 SPDX-License-Identifier: LGPL-2.0-or-later
5*/
6
7#include "storagejanitor.h"
8#include "agentmanagerinterface.h"
9#include "akonadi.h"
10#include "akonadiserver_debug.h"
11#include "akranges.h"
12#include "entities.h"
13#include "resourcemanager.h"
14#include "search/searchmanager.h"
15#include "search/searchrequest.h"
16#include "storage/collectionstatistics.h"
17#include "storage/datastore.h"
18#include "storage/dbtype.h"
19#include "storage/query.h"
20#include "storage/selectquerybuilder.h"
21#include "storage/transaction.h"
22
23#include "private/dbus_p.h"
24#include "private/externalpartstorage_p.h"
25#include "private/standarddirs_p.h"
26
27#include <QDateTime>
28#include <QDir>
29#include <QDirIterator>
30#include <QSqlError>
31#include <QSqlQuery>
32#include <QStringBuilder>
33
34#include <algorithm>
35#include <functional>
36#include <qregularexpression.h>
37
38using namespace Akonadi;
39using namespace Akonadi::Server;
40using namespace AkRanges;
41
42class StorageJanitorDataStore : public DataStore
43{
44public:
45 StorageJanitorDataStore(AkonadiServer *server, DbConfig *config)
46 : DataStore(server, config)
47 {
48 }
49};
50
51StorageJanitor::StorageJanitor(AkonadiServer *akonadi, DbConfig *dbConfig)
52 : AkThread(QStringLiteral("StorageJanitor"), QThread::IdlePriority)
53 , m_lostFoundCollectionId(-1)
54 , m_akonadi(akonadi)
55 , m_dbConfig(dbConfig)
56{
57}
58
60 : AkThread(QStringLiteral("StorageJanitor"), AkThread::NoThread)
61 , m_lostFoundCollectionId(-1)
62 , m_akonadi(nullptr)
63 , m_dbConfig(dbConfig)
64{
65 StorageJanitor::init();
66}
67
68StorageJanitor::~StorageJanitor()
69{
70 quitThread();
71}
72
73void StorageJanitor::init()
74{
75 AkThread::init();
76
77 registerTasks();
78
79 m_dataStore = std::make_unique<StorageJanitorDataStore>(m_akonadi, m_dbConfig);
80 m_dataStore->open();
81
83 conn.registerService(DBus::serviceName(DBus::StorageJanitor));
84 conn.registerObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH),
85 this,
87}
88
89void StorageJanitor::quit()
90{
92 conn.unregisterObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), QDBusConnection::UnregisterTree);
93 conn.unregisterService(DBus::serviceName(DBus::StorageJanitor));
94
95 // Make sure all children are deleted within context of this thread
96 qDeleteAll(children());
97
98 m_dataStore->close();
99
100 AkThread::quit();
101}
102
103void StorageJanitor::registerTasks()
104{
105 m_tasks = {{QStringLiteral("Looking for collections not belonging to a valid resource..."), &StorageJanitor::findOrphanedCollections},
106 {QStringLiteral("Checking collection tree consistency..."), &StorageJanitor::checkCollectionTreeConsistency},
107 {QStringLiteral("Looking for items not belonging to a valid collection..."), &StorageJanitor::findOrphanedItems},
108 {QStringLiteral("Looking for item parts not belonging to a valid item..."), &StorageJanitor::findOrphanedParts},
109 {QStringLiteral("Looking for item flags not belonging to a valid item..."), &StorageJanitor::findOrphanedPimItemFlags},
110 {QStringLiteral("Looking for duplicate item flags..."), &StorageJanitor::findDuplicateFlags},
111 {QStringLiteral("Looking for duplicate mime types..."), &StorageJanitor::findDuplicateMimeTypes},
112 {QStringLiteral("Looking for duplicate part types..."), &StorageJanitor::findDuplicatePartTypes},
113 {QStringLiteral("Looking for duplicate tag types..."), &StorageJanitor::findDuplicateTagTypes},
114 {QStringLiteral("Looking for overlapping external parts..."), &StorageJanitor::findOverlappingParts},
115 {QStringLiteral("Verifying external parts..."), &StorageJanitor::verifyExternalParts},
116 {QStringLiteral("Checking size threshold changes..."), &StorageJanitor::checkSizeTreshold},
117 {QStringLiteral("Looking for dirty objects..."), &StorageJanitor::findDirtyObjects},
118 {QStringLiteral("Looking for rid-duplicates not matching the content mime-type of the parent collection"), &StorageJanitor::findRIDDuplicates},
119 {QStringLiteral("Migrating parts to new cache hierarchy..."), &StorageJanitor::migrateToLevelledCacheHierarchy},
120 {QStringLiteral("Making sure virtual search resource and collections exist"), &StorageJanitor::ensureSearchCollection}};
121
122 // Tasks that require a valid Akonadi instance
123 if (m_akonadi) {
124 m_tasks += {{QStringLiteral("Looking for resources in the DB not matching a configured resource..."), &StorageJanitor::findOrphanedResources},
125 {QStringLiteral("Checking search index consistency..."), &StorageJanitor::findOrphanSearchIndexEntries},
126 {QStringLiteral("Flushing collection statistics memory cache..."), &StorageJanitor::expireCollectionStatisticsCache}};
127 }
128
129 /* TODO some ideas for further checks:
130 * the collection tree is non-cyclic
131 * content type constraints of collections are not violated
132 * find unused flags
133 * find unused mimetypes
134 * check for dead entries in relation tables
135 * check if part size matches file size
136 */
137}
138
139void StorageJanitor::check() // implementation of `akonadictl fsck`
140{
141 m_lostFoundCollectionId = -1; // start with a fresh one each time
142
143 for (const auto &[idx, task] : m_tasks | Views::enumerate(1)) {
144 inform(QStringLiteral("%1/%2 %3").arg(idx, 2).arg(m_tasks.size()).arg(task.name));
145 std::invoke(task.func, this);
146 }
147
148 inform("Consistency check done.");
149
150 Q_EMIT done();
151}
152
153qint64 StorageJanitor::lostAndFoundCollection()
154{
155 if (m_lostFoundCollectionId > 0) {
156 return m_lostFoundCollectionId;
157 }
158
159 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR LOST+FOUND"));
160 Resource lfRes = Resource::retrieveByName(m_dataStore.get(), QStringLiteral("akonadi_lost+found_resource"));
161 if (!lfRes.isValid()) {
162 lfRes.setName(QStringLiteral("akonadi_lost+found_resource"));
163 if (!lfRes.insert(m_dataStore.get())) {
164 qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found resource!";
165 }
166 }
167
168 Collection lfRoot;
169 SelectQueryBuilder<Collection> qb(m_dataStore.get());
170 qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, lfRes.id());
171 qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant());
172 if (!qb.exec()) {
173 qCCritical(AKONADISERVER_LOG) << "Failed to query top level collections";
174 return -1;
175 }
176 const Collection::List cols = qb.result();
177 if (cols.size() > 1) {
178 qCCritical(AKONADISERVER_LOG) << "More than one top-level lost+found collection!?";
179 } else if (cols.size() == 1) {
180 lfRoot = cols.first();
181 } else {
182 lfRoot.setName(QStringLiteral("lost+found"));
183 lfRoot.setResourceId(lfRes.id());
184 lfRoot.setCachePolicyLocalParts(QStringLiteral("ALL"));
185 lfRoot.setCachePolicyCacheTimeout(-1);
186 lfRoot.setCachePolicyInherit(false);
187 if (!lfRoot.insert(m_dataStore.get())) {
188 qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found root.";
189 }
190 if (m_akonadi) {
191 m_dataStore->notificationCollector()->collectionAdded(lfRoot, lfRes.name().toUtf8());
192 }
193 }
194
195 Collection lfCol;
196 lfCol.setName(QDateTime::currentDateTime().toString(QStringLiteral("yyyy-MM-dd hh:mm:ss")));
197 lfCol.setResourceId(lfRes.id());
198 lfCol.setParentId(lfRoot.id());
199 if (!lfCol.insert(m_dataStore.get())) {
200 qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found collection!";
201 }
202
203 const auto retrieveAll = MimeType::retrieveAll(m_dataStore.get());
204 for (const MimeType &mt : retrieveAll) {
205 lfCol.addMimeType(m_dataStore.get(), mt);
206 }
207
208 if (m_akonadi) {
209 m_dataStore->notificationCollector()->collectionAdded(lfCol, lfRes.name().toUtf8());
210 }
211
212 transaction.commit();
213 m_lostFoundCollectionId = lfCol.id();
214 return m_lostFoundCollectionId;
215}
216
217void StorageJanitor::findOrphanedResources()
218{
219 SelectQueryBuilder<Resource> qbres(m_dataStore.get());
220 OrgFreedesktopAkonadiAgentManagerInterface iface(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus(), this);
221 if (!iface.isValid()) {
222 inform(QStringLiteral("ERROR: Couldn't talk to %1").arg(DBus::Control));
223 return;
224 }
225 const QStringList knownResources = iface.agentInstances();
226 if (knownResources.isEmpty()) {
227 inform(QStringLiteral("ERROR: no known resources. This must be a mistake?"));
228 return;
229 }
230 qbres.addValueCondition(Resource::nameFullColumnName(), Query::NotIn, QVariant(knownResources));
231 qbres.addValueCondition(Resource::idFullColumnName(), Query::NotEquals, 1); // skip akonadi_search_resource
232 if (!qbres.exec()) {
233 inform("Failed to query known resources, skipping test");
234 return;
235 }
236 // qCDebug(AKONADISERVER_LOG) << "SQL:" << qbres.query().lastQuery();
237 const Resource::List orphanResources = qbres.result();
238 const int orphanResourcesSize(orphanResources.size());
239 if (orphanResourcesSize > 0) {
240 QStringList resourceNames;
241 resourceNames.reserve(orphanResourcesSize);
242 for (const Resource &resource : orphanResources) {
243 resourceNames.append(resource.name());
244 }
245 inform(QStringLiteral("Found %1 orphan resources: %2").arg(orphanResourcesSize).arg(resourceNames.join(QLatin1Char(','))));
246 for (const QString &resourceName : std::as_const(resourceNames)) {
247 inform(QStringLiteral("Removing resource %1").arg(resourceName));
248 m_akonadi->resourceManager().removeResourceInstance(resourceName);
249 }
250 }
251}
252
253void StorageJanitor::findOrphanedCollections()
254{
255 SelectQueryBuilder<Collection> qb(m_dataStore.get());
256 qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
257 qb.addValueCondition(Resource::idFullColumnName(), Query::Is, QVariant());
258
259 if (!qb.exec()) {
260 inform("Failed to query orphaned collections, skipping test");
261 return;
262 }
263 const Collection::List orphans = qb.result();
264 if (!orphans.isEmpty()) {
265 inform(QLatin1StringView("Found ") + QString::number(orphans.size()) + QLatin1StringView(" orphan collections."));
266 // TODO: attach to lost+found resource
267 }
268}
269
270void StorageJanitor::checkCollectionTreeConsistency()
271{
272 const Collection::List cols = Collection::retrieveAll(m_dataStore.get());
273 std::for_each(cols.begin(), cols.end(), [this](const Collection &col) {
274 checkPathToRoot(col);
275 });
276}
277
278void StorageJanitor::checkPathToRoot(const Collection &col)
279{
280 if (col.parentId() == 0) {
281 return;
282 }
283 const Collection parent = col.parent(m_dataStore.get());
284 if (!parent.isValid()) {
285 inform(QLatin1StringView("Collection \"") + col.name() + QLatin1StringView("\" (id: ") + QString::number(col.id())
286 + QLatin1StringView(") has no valid parent."));
287 // TODO fix that by attaching to a top-level lost+found folder
288 return;
289 }
290
291 if (col.resourceId() != parent.resourceId()) {
292 inform(QLatin1StringView("Collection \"") + col.name() + QLatin1StringView("\" (id: ") + QString::number(col.id())
293 + QLatin1StringView(") belongs to a different resource than its parent."));
294 // can/should we actually fix that?
295 }
296
297 checkPathToRoot(parent);
298}
299
300void StorageJanitor::findOrphanedItems()
301{
302 SelectQueryBuilder<PimItem> qb(m_dataStore.get());
303 qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
304 qb.addValueCondition(Collection::idFullColumnName(), Query::Is, QVariant());
305 if (!qb.exec()) {
306 inform("Failed to query orphaned items, skipping test");
307 return;
308 }
309 const PimItem::List orphans = qb.result();
310 if (!orphans.isEmpty()) {
311 inform(QLatin1StringView("Found ") + QString::number(orphans.size()) + QLatin1StringView(" orphan items."));
312 // Attach to lost+found collection
313 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR ORPHANS"));
314 QueryBuilder qb(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Update);
315 qint64 col = lostAndFoundCollection();
316 if (col == -1) {
317 return;
318 }
319 qb.setColumnValue(PimItem::collectionIdColumn(), col);
320 qb.addValueCondition(PimItem::idFullColumnName(),
321 Query::In,
322 orphans | Views::transform([](const auto &item) {
323 return item.id();
324 }) | Actions::toQList);
325 if (qb.exec() && transaction.commit()) {
326 inform(QLatin1StringView("Moved orphan items to collection ") + QString::number(col));
327 } else {
328 inform(QLatin1StringView("Error moving orphan items to collection ") + QString::number(col) + QLatin1StringView(" : ")
329 + qb.query().lastError().text());
330 }
331 }
332}
333
334void StorageJanitor::findOrphanedParts()
335{
336 SelectQueryBuilder<Part> qb(m_dataStore.get());
337 qb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
338 qb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
339 if (!qb.exec()) {
340 inform("Failed to query orphaned parts, skipping test");
341 return;
342 }
343 const Part::List orphans = qb.result();
344 if (!orphans.isEmpty()) {
345 inform(QLatin1StringView("Found ") + QString::number(orphans.size()) + QLatin1StringView(" orphan parts."));
346 // TODO: create lost+found items for those? delete?
347 }
348}
349
350void StorageJanitor::findOrphanedPimItemFlags()
351{
352 QueryBuilder sqb(m_dataStore.get(), PimItemFlagRelation::tableName(), QueryBuilder::Select);
353 sqb.addColumn(PimItemFlagRelation::leftFullColumnName());
354 sqb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), PimItemFlagRelation::leftFullColumnName(), PimItem::idFullColumnName());
355 sqb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
356 if (!sqb.exec()) {
357 inform("Failed to query orphaned item flags, skipping test");
358 return;
359 }
360
362 while (sqb.query().next()) {
363 ids.append(sqb.query().value(0).toInt());
364 }
365 sqb.query().finish();
366 if (!ids.empty()) {
367 QueryBuilder qb(m_dataStore.get(), PimItemFlagRelation::tableName(), QueryBuilder::Delete);
368 qb.addValueCondition(PimItemFlagRelation::leftFullColumnName(), Query::In, ids);
369 if (!qb.exec()) {
370 qCCritical(AKONADISERVER_LOG) << "Error:" << qb.query().lastError().text();
371 return;
372 }
373
374 inform(QLatin1StringView("Found and deleted ") + QString::number(ids.size()) + QLatin1StringView(" orphan pim item flags."));
375 }
376}
377
378struct RelationDesc {
379 QString tableName;
380 QString deduplEntityIdColumnName;
381};
382
383template<typename DeduplEntity>
384std::optional<int> findDuplicatesImpl(DataStore *dataStore, const QString &nameColumn, const RelationDesc &relation)
385{
386 QueryBuilder sqb(dataStore, DeduplEntity::tableName(), QueryBuilder::Select);
387 sqb.addColumns({DeduplEntity::idColumn(), nameColumn});
388 sqb.addSortColumn(DeduplEntity::idColumn());
389 if (!sqb.exec()) {
390 return std::nullopt;
391 }
392
394 while (sqb.query().next()) {
395 const auto id = sqb.query().value(0).toLongLong();
396 const auto name = sqb.query().value(1).toString();
397
398 auto it = duplicates.find(name.trimmed());
399 if (it == duplicates.end()) {
400 it = duplicates.insert(name.trimmed(), QVariantList{});
401 }
402 it->push_back(id);
403 }
404
405 int removed = 0;
406 for (const auto &[duplicateName, duplicateIds] : duplicates.asKeyValueRange()) {
407 if (duplicateIds.size() <= 1) {
408 // Not duplicated
409 continue;
410 }
411
412 Transaction transaction(dataStore, QStringLiteral("StorageJanitor deduplicate %1 %2").arg(DeduplEntity::tableName(), duplicateName));
413
414 // Update all relations with duplicated entity to use the lowest entity ID, so we can remove the
415 // duplicates afterwards
416 const auto firstId = duplicateIds.takeFirst();
417
418 QueryBuilder updateQb(dataStore, relation.tableName, QueryBuilder::Update);
419 updateQb.setColumnValue(relation.deduplEntityIdColumnName, firstId);
420 updateQb.addValueCondition(relation.deduplEntityIdColumnName, Query::In, duplicateIds);
421 if (!updateQb.exec()) {
422 continue;
423 }
424
425 // Remove the duplicated entities
426 QueryBuilder removeQb(dataStore, DeduplEntity::tableName(), QueryBuilder::Delete);
427 removeQb.addValueCondition(DeduplEntity::idColumn(), Query::In, duplicateIds);
428 if (!removeQb.exec()) {
429 continue;
430 }
431
432 ++removed;
433
434 transaction.commit();
435 }
436
437 return removed;
438}
439
440void StorageJanitor::findDuplicateFlags()
441{
442 const auto removed =
443 findDuplicatesImpl<Flag>(m_dataStore.get(), Flag::nameFullColumnName(), {PimItemFlagRelation::tableName(), PimItemFlagRelation::rightFullColumnName()});
444 if (removed) {
445 inform(u"Removed " % QString::number(*removed) % u" duplicate item flags");
446 } else {
447 inform("Error while trying to remove duplicate Flags");
448 }
449}
450
451void StorageJanitor::findDuplicateMimeTypes()
452{
453 const auto removed =
454 findDuplicatesImpl<MimeType>(m_dataStore.get(), MimeType::nameFullColumnName(), {PimItem::tableName(), PimItem::mimeTypeIdFullColumnName()});
455 if (removed) {
456 inform(u"Removed " % QString::number(*removed) % u" duplicate mime types");
457 } else {
458 inform("Error while trying to remove duplicate MimeTypes");
459 }
460}
461
462void StorageJanitor::findDuplicatePartTypes()
463{
464 // Good thing that SQL is ANSI/ISO standardized...
465 QString nameColumn;
466 if (DbType::type(m_dataStore->database()) == DbType::MySQL) {
467 nameColumn = QStringLiteral("CONCAT_WS(':', %1, %2) AS name");
468 } else {
469 nameColumn = QStringLiteral("(%1 || ':' || %2) AS name");
470 }
471
472 const auto removed = findDuplicatesImpl<PartType>(m_dataStore.get(),
473 nameColumn.arg(PartType::nsFullColumnName(), PartType::nameFullColumnName()),
474 {Part::tableName(), Part::partTypeIdFullColumnName()});
475 if (removed) {
476 inform(u"Removed " % QString::number(*removed) % u" duplicate part types");
477 } else {
478 inform("Error while trying to remove duplicate PartTypes");
479 }
480}
481
482void StorageJanitor::findDuplicateTagTypes()
483{
484 const auto removed = findDuplicatesImpl<TagType>(m_dataStore.get(), TagType::nameFullColumnName(), {Tag::tableName(), Tag::typeIdFullColumnName()});
485 if (removed) {
486 inform(u"Removed " % QString::number(*removed) % u" duplicate tag types");
487 } else {
488 inform("Error while trying to remove duplicate TagTypes");
489 }
490}
491
492void StorageJanitor::findOverlappingParts()
493{
494 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
495 qb.addColumn(Part::dataColumn());
496 qb.addColumn(QLatin1StringView("count(") + Part::idColumn() + QLatin1StringView(") as cnt"));
497 qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
498 qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
499 qb.addGroupColumn(Part::dataColumn());
500 qb.addValueCondition(QLatin1StringView("count(") + Part::idColumn() + QLatin1StringView(")"), Query::Greater, 1, QueryBuilder::HavingCondition);
501 if (!qb.exec()) {
502 inform("Failed to query overlapping parts, skipping test");
503 return;
504 }
505
506 int count = 0;
507 while (qb.query().next()) {
508 ++count;
509 inform(QLatin1StringView("Found overlapping part data: ") + qb.query().value(0).toString());
510 // TODO: uh oh, this is bad, how do we recover from that?
511 }
512 qb.query().finish();
513
514 if (count > 0) {
515 inform(QLatin1StringView("Found ") + QString::number(count) + QLatin1StringView(" overlapping parts - bad."));
516 }
517}
518
519void StorageJanitor::verifyExternalParts()
520{
521 QSet<QString> existingFiles;
522 QSet<QString> usedFiles;
523
524 // list all files
525 const QString dataDir = StandardDirs::saveDir("data", QStringLiteral("file_db_data"));
527 while (it.hasNext()) {
528 existingFiles.insert(it.next());
529 }
530 existingFiles.remove(dataDir + QDir::separator() + QLatin1Char('.'));
531 existingFiles.remove(dataDir + QDir::separator() + QLatin1StringView(".."));
532 inform(QLatin1StringView("Found ") + QString::number(existingFiles.size()) + QLatin1StringView(" external files."));
533
534 // list all parts from the db which claim to have an associated file
535 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
536 qb.addColumn(Part::dataColumn());
537 qb.addColumn(Part::pimItemIdColumn());
538 qb.addColumn(Part::idColumn());
539 qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
540 qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
541 if (!qb.exec()) {
542 inform("Failed to query existing parts, skipping test");
543 return;
544 }
545 while (qb.query().next()) {
546 const auto filename = qb.query().value(0).toByteArray();
547 const auto pimItemId = qb.query().value(1).value<Entity::Id>();
548 const auto partId = qb.query().value(2).value<Entity::Id>();
549 QString partPath;
550 if (!filename.isEmpty()) {
551 partPath = ExternalPartStorage::resolveAbsolutePath(filename);
552 } else {
553 partPath = ExternalPartStorage::resolveAbsolutePath(ExternalPartStorage::nameForPartId(partId));
554 }
555 if (existingFiles.contains(partPath)) {
556 usedFiles.insert(partPath);
557 } else {
558 inform(QLatin1StringView("Cleaning up missing external file: ") + partPath + QLatin1StringView(" for item: ") + QString::number(pimItemId)
559 + QLatin1StringView(" on part: ") + QString::number(partId));
560
561 Part part;
562 part.setId(partId);
563 part.setPimItemId(pimItemId);
564 part.setData(QByteArray());
565 part.setDatasize(0);
566 part.setStorage(Part::Internal);
567 part.update(m_dataStore.get());
568 }
569 }
570 qb.query().finish();
571 inform(QLatin1StringView("Found ") + QString::number(usedFiles.size()) + QLatin1StringView(" external parts."));
572
573 // see what's left and move it to lost+found
574 const QSet<QString> unreferencedFiles = existingFiles - usedFiles;
575 if (!unreferencedFiles.isEmpty()) {
576 const QString lfDir = StandardDirs::saveDir("data", QStringLiteral("file_lost+found"));
577 for (const QString &file : unreferencedFiles) {
578 inform(QLatin1StringView("Found unreferenced external file: ") + file);
579 const QFileInfo f(file);
580 QFile::rename(file, lfDir + QDir::separator() + f.fileName());
581 }
582 inform(QStringLiteral("Moved %1 unreferenced files to lost+found.").arg(unreferencedFiles.size()));
583 } else {
584 inform("Found no unreferenced external files.");
585 }
586}
587
588void StorageJanitor::findDirtyObjects()
589{
590 SelectQueryBuilder<Collection> cqb(m_dataStore.get());
591 cqb.setSubQueryMode(Query::Or);
592 cqb.addValueCondition(Collection::remoteIdColumn(), Query::Is, QVariant());
593 cqb.addValueCondition(Collection::remoteIdColumn(), Query::Equals, QString());
594 if (!cqb.exec()) {
595 inform("Failed to query collections without RID, skipping test");
596 return;
597 }
598 const Collection::List ridLessCols = cqb.result();
599 for (const Collection &col : ridLessCols) {
600 inform(QLatin1StringView("Collection \"") + col.name() + QLatin1StringView("\" (id: ") + QString::number(col.id())
601 + QLatin1StringView(") has no RID."));
602 }
603 inform(QLatin1StringView("Found ") + QString::number(ridLessCols.size()) + QLatin1StringView(" collections without RID."));
604
605 SelectQueryBuilder<PimItem> iqb1(m_dataStore.get());
606 iqb1.setSubQueryMode(Query::Or);
607 iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Is, QVariant());
608 iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, QString());
609 if (!iqb1.exec()) {
610 inform("Failed to query items without RID, skipping test");
611 return;
612 }
613 const PimItem::List ridLessItems = iqb1.result();
614 for (const PimItem &item : ridLessItems) {
615 inform(QLatin1StringView("Item \"") + QString::number(item.id()) + QLatin1StringView("\" in collection \"") + QString::number(item.collectionId())
616 + QLatin1StringView("\" has no RID."));
617 }
618 inform(QLatin1StringView("Found ") + QString::number(ridLessItems.size()) + QLatin1StringView(" items without RID."));
619
620 SelectQueryBuilder<PimItem> iqb2(m_dataStore.get());
621 iqb2.addValueCondition(PimItem::dirtyColumn(), Query::Equals, true);
622 iqb2.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
623 iqb2.addSortColumn(PimItem::idFullColumnName());
624 if (!iqb2.exec()) {
625 inform("Failed to query dirty items, skipping test");
626 return;
627 }
628 const PimItem::List dirtyItems = iqb2.result();
629 for (const PimItem &item : dirtyItems) {
630 inform(QLatin1StringView("Item \"") + QString::number(item.id()) + QLatin1StringView("\" has RID and is dirty."));
631 }
632 inform(QLatin1StringView("Found ") + QString::number(dirtyItems.size()) + QLatin1StringView(" dirty items."));
633}
634
635void StorageJanitor::findRIDDuplicates()
636{
637 QueryBuilder qb(m_dataStore.get(), Collection::tableName(), QueryBuilder::Select);
638 qb.addColumn(Collection::idColumn());
639 qb.addColumn(Collection::nameColumn());
640 qb.exec();
641
642 while (qb.query().next()) {
643 const auto colId = qb.query().value(0).value<Collection::Id>();
644 const QString name = qb.query().value(1).toString();
645 inform(QStringLiteral("Checking ") + name);
646
647 QueryBuilder duplicates(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Select);
648 duplicates.addColumn(PimItem::remoteIdColumn());
649 duplicates.addColumn(QStringLiteral("count(") + PimItem::idColumn() + QStringLiteral(") as cnt"));
650 duplicates.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
651 duplicates.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
652 duplicates.addGroupColumn(PimItem::remoteIdColumn());
653 duplicates.addValueCondition(QStringLiteral("count(") + PimItem::idColumn() + QLatin1Char(')'), Query::Greater, 1, QueryBuilder::HavingCondition);
654 duplicates.exec();
655
656 Akonadi::Server::Collection col = Akonadi::Server::Collection::retrieveById(m_dataStore.get(), colId);
657 const QList<Akonadi::Server::MimeType> contentMimeTypes = col.mimeTypes(m_dataStore.get());
658 QVariantList contentMimeTypesVariantList;
659 contentMimeTypesVariantList.reserve(contentMimeTypes.count());
660 for (const Akonadi::Server::MimeType &mimeType : contentMimeTypes) {
661 contentMimeTypesVariantList << mimeType.id();
662 }
663 while (duplicates.query().next()) {
664 const QString rid = duplicates.query().value(0).toString();
665
666 Query::Condition condition(Query::And);
667 condition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, rid);
668 condition.addValueCondition(PimItem::mimeTypeIdColumn(), Query::NotIn, contentMimeTypesVariantList);
669 condition.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
670
671 QueryBuilder items(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Select);
672 items.addColumn(PimItem::idColumn());
673 items.addCondition(condition);
674 if (!items.exec()) {
675 inform(QStringLiteral("Error while deleting duplicates: ") + items.query().lastError().text());
676 continue;
677 }
678 QVariantList itemsIds;
679 while (items.query().next()) {
680 itemsIds.push_back(items.query().value(0));
681 }
682 items.query().finish();
683 if (itemsIds.isEmpty()) {
684 // the mimetype filter may have dropped some entries from the
685 // duplicates query
686 continue;
687 }
688
689 inform(QStringLiteral("Found duplicates ") + rid);
690
691 SelectQueryBuilder<Part> parts(m_dataStore.get());
692 parts.addValueCondition(Part::pimItemIdFullColumnName(), Query::In, QVariant::fromValue(itemsIds));
693 parts.addValueCondition(Part::storageFullColumnName(), Query::Equals, static_cast<int>(Part::External));
694 if (parts.exec()) {
695 const auto partsList = parts.result();
696 for (const auto &part : partsList) {
697 bool exists = false;
698 const auto filename = ExternalPartStorage::resolveAbsolutePath(part.data(), &exists);
699 if (exists) {
700 QFile::remove(filename);
701 }
702 }
703 }
704
705 items = QueryBuilder(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Delete);
706 items.addCondition(condition);
707 if (!items.exec()) {
708 inform(QStringLiteral("Error while deleting duplicates ") + items.query().lastError().text());
709 }
710 }
711 duplicates.query().finish();
712 }
713 qb.query().finish();
714}
715
717{
718 const DbType::Type dbType = DbType::type(m_dataStore->database());
719 if (dbType == DbType::MySQL || dbType == DbType::PostgreSQL) {
720 inform("vacuuming database, that'll take some time and require a lot of temporary disk space...");
721 const auto tables = allDatabaseTables();
722 for (const QString &table : tables) {
723 inform(QStringLiteral("optimizing table %1...").arg(table));
724
725 QString queryStr;
726 if (dbType == DbType::MySQL) {
727 queryStr = QLatin1StringView("OPTIMIZE TABLE ") + table;
728 } else if (dbType == DbType::PostgreSQL) {
729 queryStr = QLatin1StringView("VACUUM FULL ANALYZE ") + table;
730 } else {
731 continue;
732 }
733 QSqlQuery q(m_dataStore->database());
734 if (!q.exec(queryStr)) {
735 qCCritical(AKONADISERVER_LOG) << "failed to optimize table" << table << ":" << q.lastError().text();
736 }
737 }
738 inform("vacuum done");
739 } else {
740 inform("Vacuum not supported for this database backend. (Sqlite backend)");
741 }
742
743 Q_EMIT done();
744}
745
746void StorageJanitor::checkSizeTreshold()
747{
748 {
749 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
750 qb.addColumn(Part::idFullColumnName());
751 qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::Internal);
752 qb.addValueCondition(Part::datasizeFullColumnName(), Query::Greater, m_dbConfig->sizeThreshold());
753 if (!qb.exec()) {
754 inform("Failed to query parts larger than threshold, skipping test");
755 return;
756 }
757
758 auto &query = qb.query();
759 inform(QStringLiteral("Found %1 parts to be moved to external files").arg(query.size()));
760
761 while (query.next()) {
762 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD"));
763 Part part = Part::retrieveById(m_dataStore.get(), query.value(0).toLongLong());
764 const QByteArray name = ExternalPartStorage::nameForPartId(part.id());
765 const QString partPath = ExternalPartStorage::resolveAbsolutePath(name);
766 QFile f(partPath);
767 if (f.exists()) {
768 qCDebug(AKONADISERVER_LOG) << "External payload file" << name << "already exists";
769 // That however is not a critical issue, since the part is not external,
770 // so we can safely overwrite it
771 }
773 qCCritical(AKONADISERVER_LOG) << "Failed to open file" << name << "for writing";
774 continue;
775 }
776 if (f.write(part.data()) != part.datasize()) {
777 qCCritical(AKONADISERVER_LOG) << "Failed to write data to payload file" << name;
778 f.remove();
779 continue;
780 }
781
782 part.setData(name);
783 part.setStorage(Part::External);
784 if (!part.update(m_dataStore.get()) || !transaction.commit()) {
785 qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
786 f.remove();
787 continue;
788 }
789
790 inform(QStringLiteral("Moved part %1 from database into external file %2").arg(part.id()).arg(QString::fromLatin1(name)));
791 }
792 }
793
794 {
795 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
796 qb.addColumn(Part::idFullColumnName());
797 qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
798 qb.addValueCondition(Part::datasizeFullColumnName(), Query::Less, DbConfig::configuredDatabase()->sizeThreshold());
799 if (!qb.exec()) {
800 inform("Failed to query parts smaller than threshold, skipping test");
801 return;
802 }
803
804 auto &query = qb.query();
805 inform(QStringLiteral("Found %1 parts to be moved to database").arg(query.size()));
806
807 while (query.next()) {
808 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD 2"));
809 Part part = Part::retrieveById(m_dataStore.get(), query.value(0).toLongLong());
810 const QString partPath = ExternalPartStorage::resolveAbsolutePath(part.data());
811 QFile f(partPath);
812 if (!f.exists()) {
813 qCCritical(AKONADISERVER_LOG) << "Part file" << part.data() << "does not exist";
814 continue;
815 }
816 if (!f.open(QIODevice::ReadOnly)) {
817 qCCritical(AKONADISERVER_LOG) << "Failed to open part file" << part.data() << "for reading";
818 continue;
819 }
820
821 part.setStorage(Part::Internal);
822 part.setData(f.readAll());
823 if (part.data().size() != part.datasize()) {
824 qCCritical(AKONADISERVER_LOG) << "Sizes of" << part.id() << "data don't match";
825 continue;
826 }
827 if (!part.update(m_dataStore.get()) || !transaction.commit()) {
828 qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
829 continue;
830 }
831
832 f.close();
833 f.remove();
834 inform(QStringLiteral("Moved part %1 from external file into database").arg(part.id()));
835 }
836 }
837}
838
839void StorageJanitor::migrateToLevelledCacheHierarchy()
840{
841 /// First, check whether that's still necessary
842 const QString db_data = StandardDirs::saveDir("data", QStringLiteral("file_db_data"));
843 const auto entries = QDir(db_data).entryInfoList(QDir::Files | QDir::NoDotAndDotDot);
844 if (entries.isEmpty()) {
845 inform("No external parts in legacy location, skipping migration");
846 return;
847 }
848
849 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
850 qb.addColumn(Part::idColumn());
851 qb.addColumn(Part::dataColumn());
852 qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
853 if (!qb.exec()) {
854 inform("Failed to query external payload parts, skipping test");
855 return;
856 }
857
858 auto &query = qb.query();
859 while (query.next()) {
860 const qint64 id = query.value(0).toLongLong();
861 const QByteArray data = query.value(1).toByteArray();
862 const QString fileName = QString::fromUtf8(data);
863 bool oldExists = false;
864 bool newExists = false;
865 // Resolve the current path
866 const QString currentPath = ExternalPartStorage::resolveAbsolutePath(fileName, &oldExists);
867 // Resolve the new path with legacy fallback disabled, so that it always
868 // returns the new levelled-cache path, even when the old one exists
869 const QString newPath = ExternalPartStorage::resolveAbsolutePath(fileName, &newExists, false);
870 if (!oldExists) {
871 qCCritical(AKONADISERVER_LOG) << "Old payload part does not exist, skipping part" << fileName;
872 continue;
873 }
874 if (currentPath != newPath) {
875 if (newExists) {
876 qCCritical(AKONADISERVER_LOG) << "Part is in legacy location, but the destination file already exists, skipping part" << fileName;
877 continue;
878 }
879
880 QFile f(currentPath);
881 if (!f.rename(newPath)) {
882 qCCritical(AKONADISERVER_LOG) << "Failed to move part from" << currentPath << " to " << newPath << ":" << f.errorString();
883 continue;
884 }
885 inform(QStringLiteral("Migrated part %1 to new levelled cache").arg(id));
886 }
887 }
888}
889
890void StorageJanitor::findOrphanSearchIndexEntries()
891{
892 QueryBuilder qb(m_dataStore.get(), Collection::tableName(), QueryBuilder::Select);
893 qb.addSortColumn(Collection::idColumn(), Query::Ascending);
894 qb.addColumn(Collection::idColumn());
895 qb.addColumn(Collection::isVirtualColumn());
896 if (!qb.exec()) {
897 inform("Failed to query collections, skipping test");
898 return;
899 }
900
901 QDBusInterface iface(DBus::agentServiceName(QStringLiteral("akonadi_indexing_agent"), DBus::Agent),
902 QStringLiteral("/"),
903 QStringLiteral("org.freedesktop.Akonadi.Indexer"),
905 if (!iface.isValid()) {
906 inform("Akonadi Indexing Agent is not running, skipping test");
907 return;
908 }
909
910 auto &query = qb.query();
911 while (query.next()) {
912 const qint64 colId = query.value(0).toLongLong();
913 // Skip virtual collections, they are not indexed
914 if (query.value(1).toBool()) {
915 inform(QStringLiteral("Skipping virtual Collection %1").arg(colId));
916 continue;
917 }
918
919 inform(QStringLiteral("Checking Collection %1 search index...").arg(colId));
920 SearchRequest req("StorageJanitor", m_akonadi->searchManager(), m_akonadi->agentSearchManager());
921 req.setStoreResults(true);
922 req.setCollections({colId});
923 req.setRemoteSearch(false);
924 req.setQuery(QStringLiteral("{ }")); // empty query to match all
925 QStringList mts;
926 Collection col;
927 col.setId(colId);
928 const auto colMts = col.mimeTypes();
929 if (colMts.isEmpty()) {
930 // No mimetypes means we don't know which search store to look into,
931 // skip it.
932 continue;
933 }
934 mts.reserve(colMts.count());
935 for (const auto &mt : colMts) {
936 mts << mt.name();
937 }
938 req.setMimeTypes(mts);
939 req.exec();
940 auto searchResults = req.results();
941
942 QueryBuilder iqb(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Select);
943 iqb.addColumn(PimItem::idColumn());
944 iqb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
945 if (!iqb.exec()) {
946 inform(QStringLiteral("Failed to query items in collection %1").arg(colId));
947 continue;
948 }
949
950 auto &itemQuery = iqb.query();
951 while (itemQuery.next()) {
952 searchResults.remove(itemQuery.value(0).toLongLong());
953 }
954
955 if (!searchResults.isEmpty()) {
956 inform(QStringLiteral("Collection %1 search index contains %2 orphan items. Scheduling reindexing").arg(colId).arg(searchResults.count()));
957 iface.call(QDBus::NoBlock, QStringLiteral("reindexCollection"), colId);
958 }
959 }
960}
961
962void StorageJanitor::ensureSearchCollection()
963{
964 static const auto searchResourceName = QStringLiteral("akonadi_search_resource");
965
966 auto searchResource = Resource::retrieveByName(m_dataStore.get(), searchResourceName);
967 if (!searchResource.isValid()) {
968 searchResource.setName(searchResourceName);
969 searchResource.setIsVirtual(true);
970 if (!searchResource.insert(m_dataStore.get())) {
971 inform(QStringLiteral("Failed to create Search resource."));
972 return;
973 }
974 }
975
976 auto searchCols = Collection::retrieveFiltered(m_dataStore.get(), Collection::resourceIdColumn(), searchResource.id());
977 if (searchCols.isEmpty()) {
978 Collection searchCol;
979 searchCol.setId(1);
980 searchCol.setName(QStringLiteral("Search"));
981 searchCol.setResource(searchResource);
982 searchCol.setIndexPref(Collection::False);
983 searchCol.setIsVirtual(true);
984 if (!searchCol.insert(m_dataStore.get())) {
985 inform(QStringLiteral("Failed to create Search Collection"));
986 return;
987 }
988 }
989}
990
991void StorageJanitor::expireCollectionStatisticsCache()
992{
993 m_akonadi->collectionStatistics().expireCache();
994}
995
996void StorageJanitor::inform(const char *msg)
997{
998 inform(QLatin1StringView(msg));
999}
1000
1001void StorageJanitor::inform(const QString &msg)
1002{
1003 qCDebug(AKONADISERVER_LOG) << msg;
1004 Q_EMIT information(msg);
1005}
1006
1007#include "moc_storagejanitor.cpp"
Represents a collection of PIM items.
Definition collection.h:62
qint64 Id
Describes the unique id type.
Definition collection.h:79
void setName(const QString &name)
Sets the i18n'ed name of the collection.
void setResource(const QString &identifier)
Sets the identifier of the resource owning the collection.
void setId(Id identifier)
Sets the unique identifier of the collection.
This class handles all the database access.
Definition datastore.h:95
A base class that provides an unique access layer to configuration and initialization of different da...
Definition dbconfig.h:21
virtual qint64 sizeThreshold() const
Payload data bigger than this value will be stored in separate files, instead of the database.
Definition dbconfig.cpp:138
static DbConfig * configuredDatabase()
Returns the DbConfig instance for the database the user has configured.
Definition dbconfig.cpp:77
Helper class to construct arbitrary SQL queries.
@ HavingCondition
add condition to HAVING part of the query NOTE: only supported for SELECT queries
@ LeftJoin
NOTE: only supported for SELECT queries.
Represents a WHERE condition tree.
Definition query.h:62
Helper class for creating and executing database SELECT queries.
Q_SCRIPTABLE Q_NOREPLY void vacuum()
Triggers a vacuuming of the database, that is compacting of unused space.
StorageJanitor(AkonadiServer *mAkonadi, DbConfig *config=DbConfig::configuredDatabase())
Use AkThread::create() to create and start a new StorageJanitor thread.
Q_SCRIPTABLE Q_NOREPLY void check()
Triggers a consistency check of the internal storage.
Q_SCRIPTABLE void information(const QString &msg)
Sends informational messages to a possible UI for this.
Helper class for DataStore transaction handling.
Definition transaction.h:23
QString name() const
void setName(QString value)
Type
Supported database types.
Definition dbtype.h:19
Type type(const QSqlDatabase &db)
Returns the type of the given database object.
Definition dbtype.cpp:11
Helper integration between Akonadi and Qt.
char * toString(const EngineQuery &query)
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
KCALUTILS_EXPORT QString mimeType()
QString name(StandardAction id)
QDateTime currentDateTime()
bool registerObject(const QString &path, QObject *object, RegisterOptions options)
bool registerService(const QString &serviceName)
QDBusConnection sessionBus()
void unregisterObject(const QString &path, UnregisterMode mode)
bool unregisterService(const QString &serviceName)
QFileInfoList entryInfoList(Filters filters, SortFlags sort) const const
QChar separator()
bool remove()
bool rename(const QString &newName)
QString toString() const const
void append(QList< T > &&value)
iterator begin()
qsizetype count() const const
bool empty() const const
iterator end()
T & first()
bool isEmpty() const const
void reserve(qsizetype size)
qsizetype size() const const
T value(qsizetype i) const const
auto asKeyValueRange() &
iterator end()
iterator find(const Key &key)
iterator insert(const Key &key, const T &value)
T value(const Key &key, const T &defaultValue) const const
Q_EMITQ_EMIT
const QObjectList & children() const const
QObject * parent() const const
bool contains(const QSet< T > &other) const const
iterator insert(const T &value)
bool isEmpty() const const
bool remove(const T &value)
qsizetype size() const const
QString text() const const
bool exec()
QSqlError lastError() const const
QString arg(Args &&... args) const const
QString fromLatin1(QByteArrayView str)
QString fromUtf8(QByteArrayView str)
QString number(double n, char format, int precision)
QString & remove(QChar ch, Qt::CaseSensitivity cs)
QByteArray toUtf8() const const
QString trimmed() const const
QString join(QChar separator) const const
QVariant fromValue(T &&value)
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Fri Dec 6 2024 12:01:00 by doxygen 1.12.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.