Akonadi

storagejanitor.cpp
1/*
2 SPDX-FileCopyrightText: 2011 Volker Krause <vkrause@kde.org>
3
4 SPDX-License-Identifier: LGPL-2.0-or-later
5*/
6
7#include "storagejanitor.h"
8#include "agentmanagerinterface.h"
9#include "akonadi.h"
10#include "akonadiserver_debug.h"
11#include "akranges.h"
12#include "entities.h"
13#include "resourcemanager.h"
14#include "search/searchmanager.h"
15#include "search/searchrequest.h"
16#include "storage/collectionstatistics.h"
17#include "storage/datastore.h"
18#include "storage/dbtype.h"
19#include "storage/query.h"
20#include "storage/selectquerybuilder.h"
21#include "storage/transaction.h"
22
23#include "private/dbus_p.h"
24#include "private/externalpartstorage_p.h"
25#include "private/standarddirs_p.h"
26
27#include <QDateTime>
28#include <QDir>
29#include <QDirIterator>
30#include <QSqlError>
31#include <QSqlQuery>
32#include <QStringBuilder>
33
34#include <algorithm>
35#include <functional>
36#include <qregularexpression.h>
37
38using namespace Akonadi;
39using namespace Akonadi::Server;
40using namespace AkRanges;
41
42class StorageJanitorDataStore : public DataStore
43{
44public:
45 StorageJanitorDataStore(AkonadiServer *server, DbConfig *config)
46 : DataStore(server, config)
47 {
48 }
49};
50
51StorageJanitor::StorageJanitor(AkonadiServer *akonadi, DbConfig *dbConfig)
52 : AkThread(QStringLiteral("StorageJanitor"), QThread::IdlePriority)
53 , m_lostFoundCollectionId(-1)
54 , m_akonadi(akonadi)
55 , m_dbConfig(dbConfig)
56{
57}
58
60 : AkThread(QStringLiteral("StorageJanitor"), AkThread::NoThread)
61 , m_lostFoundCollectionId(-1)
62 , m_akonadi(nullptr)
63 , m_dbConfig(dbConfig)
64{
65 StorageJanitor::init();
66}
67
68StorageJanitor::~StorageJanitor()
69{
70 quitThread();
71}
72
73void StorageJanitor::init()
74{
75 AkThread::init();
76
77 registerTasks();
78
79 m_dataStore = std::make_unique<StorageJanitorDataStore>(m_akonadi, m_dbConfig);
80 m_dataStore->open();
81
83 conn.registerService(DBus::serviceName(DBus::StorageJanitor));
84 conn.registerObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH),
85 this,
87}
88
89void StorageJanitor::quit()
90{
93 conn.unregisterService(DBus::serviceName(DBus::StorageJanitor));
94
95 // Make sure all children are deleted within context of this thread
97
98 m_dataStore->close();
99
100 AkThread::quit();
101}
102
103void StorageJanitor::registerTasks()
104{
105 m_tasks = {{QStringLiteral("Looking for collections not belonging to a valid resource..."), &StorageJanitor::findOrphanedCollections},
106 {QStringLiteral("Checking collection tree consistency..."), &StorageJanitor::checkCollectionTreeConsistency},
107 {QStringLiteral("Looking for items not belonging to a valid collection..."), &StorageJanitor::findOrphanedItems},
108 {QStringLiteral("Looking for item parts not belonging to a valid item..."), &StorageJanitor::findOrphanedParts},
109 {QStringLiteral("Looking for item flags not belonging to a valid item..."), &StorageJanitor::findOrphanedPimItemFlags},
110 {QStringLiteral("Looking for duplicate item flags..."), &StorageJanitor::findDuplicateFlags},
111 {QStringLiteral("Looking for duplicate mime types..."), &StorageJanitor::findDuplicateMimeTypes},
112 {QStringLiteral("Looking for duplicate part types..."), &StorageJanitor::findDuplicatePartTypes},
113 {QStringLiteral("Looking for duplicate tag types..."), &StorageJanitor::findDuplicateTagTypes},
114 {QStringLiteral("Looking for duplicate relation types..."), &StorageJanitor::findDuplicateRelationTypes},
115 {QStringLiteral("Looking for overlapping external parts..."), &StorageJanitor::findOverlappingParts},
116 {QStringLiteral("Verifying external parts..."), &StorageJanitor::verifyExternalParts},
117 {QStringLiteral("Checking size threshold changes..."), &StorageJanitor::checkSizeTreshold},
118 {QStringLiteral("Looking for dirty objects..."), &StorageJanitor::findDirtyObjects},
119 {QStringLiteral("Looking for rid-duplicates not matching the content mime-type of the parent collection"), &StorageJanitor::findRIDDuplicates},
120 {QStringLiteral("Migrating parts to new cache hierarchy..."), &StorageJanitor::migrateToLevelledCacheHierarchy},
121 {QStringLiteral("Making sure virtual search resource and collections exist"), &StorageJanitor::ensureSearchCollection}};
122
123 // Tasks that require a valid Akonadi instance
124 if (m_akonadi) {
125 m_tasks += {{QStringLiteral("Looking for resources in the DB not matching a configured resource..."), &StorageJanitor::findOrphanedResources},
126 {QStringLiteral("Checking search index consistency..."), &StorageJanitor::findOrphanSearchIndexEntries},
127 {QStringLiteral("Flushing collection statistics memory cache..."), &StorageJanitor::expireCollectionStatisticsCache}};
128 }
129
130 /* TODO some ideas for further checks:
131 * the collection tree is non-cyclic
132 * content type constraints of collections are not violated
133 * find unused flags
134 * find unused mimetypes
135 * check for dead entries in relation tables
136 * check if part size matches file size
137 */
138}
139
140void StorageJanitor::check() // implementation of `akonadictl fsck`
141{
142 m_lostFoundCollectionId = -1; // start with a fresh one each time
143
144 for (const auto &[idx, task] : m_tasks | Views::enumerate(1)) {
145 inform(QStringLiteral("%1/%2 %3").arg(idx, 2).arg(m_tasks.size()).arg(task.name));
146 std::invoke(task.func, this);
147 }
148
149 inform("Consistency check done.");
150
151 Q_EMIT done();
152}
153
154qint64 StorageJanitor::lostAndFoundCollection()
155{
156 if (m_lostFoundCollectionId > 0) {
157 return m_lostFoundCollectionId;
158 }
159
160 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR LOST+FOUND"));
161 Resource lfRes = Resource::retrieveByName(m_dataStore.get(), QStringLiteral("akonadi_lost+found_resource"));
162 if (!lfRes.isValid()) {
163 lfRes.setName(QStringLiteral("akonadi_lost+found_resource"));
164 if (!lfRes.insert(m_dataStore.get())) {
165 qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found resource!";
166 }
167 }
168
170 SelectQueryBuilder<Collection> qb(m_dataStore.get());
171 qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, lfRes.id());
172 qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant());
173 if (!qb.exec()) {
174 qCCritical(AKONADISERVER_LOG) << "Failed to query top level collections";
175 return -1;
176 }
177 const Collection::List cols = qb.result();
178 if (cols.size() > 1) {
179 qCCritical(AKONADISERVER_LOG) << "More than one top-level lost+found collection!?";
180 } else if (cols.size() == 1) {
181 lfRoot = cols.first();
182 } else {
183 lfRoot.setName(QStringLiteral("lost+found"));
184 lfRoot.setResourceId(lfRes.id());
185 lfRoot.setCachePolicyLocalParts(QStringLiteral("ALL"));
186 lfRoot.setCachePolicyCacheTimeout(-1);
187 lfRoot.setCachePolicyInherit(false);
188 if (!lfRoot.insert(m_dataStore.get())) {
189 qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found root.";
190 }
191 if (m_akonadi) {
192 m_dataStore->notificationCollector()->collectionAdded(lfRoot, lfRes.name().toUtf8());
193 }
194 }
195
197 lfCol.setName(QDateTime::currentDateTime().toString(QStringLiteral("yyyy-MM-dd hh:mm:ss")));
198 lfCol.setResourceId(lfRes.id());
199 lfCol.setParentId(lfRoot.id());
200 if (!lfCol.insert(m_dataStore.get())) {
201 qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found collection!";
202 }
203
204 const auto retrieveAll = MimeType::retrieveAll(m_dataStore.get());
205 for (const MimeType &mt : retrieveAll) {
206 lfCol.addMimeType(m_dataStore.get(), mt);
207 }
208
209 if (m_akonadi) {
210 m_dataStore->notificationCollector()->collectionAdded(lfCol, lfRes.name().toUtf8());
211 }
212
213 transaction.commit();
214 m_lostFoundCollectionId = lfCol.id();
215 return m_lostFoundCollectionId;
216}
217
218void StorageJanitor::findOrphanedResources()
219{
220 SelectQueryBuilder<Resource> qbres(m_dataStore.get());
221 OrgFreedesktopAkonadiAgentManagerInterface iface(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus(), this);
222 if (!iface.isValid()) {
223 inform(QStringLiteral("ERROR: Couldn't talk to %1").arg(DBus::Control));
224 return;
225 }
226 const QStringList knownResources = iface.agentInstances();
227 if (knownResources.isEmpty()) {
228 inform(QStringLiteral("ERROR: no known resources. This must be a mistake?"));
229 return;
230 }
231 qbres.addValueCondition(Resource::nameFullColumnName(), Query::NotIn, QVariant(knownResources));
232 qbres.addValueCondition(Resource::idFullColumnName(), Query::NotEquals, 1); // skip akonadi_search_resource
233 if (!qbres.exec()) {
234 inform("Failed to query known resources, skipping test");
235 return;
236 }
237 // qCDebug(AKONADISERVER_LOG) << "SQL:" << qbres.query().lastQuery();
238 const Resource::List orphanResources = qbres.result();
239 const int orphanResourcesSize(orphanResources.size());
240 if (orphanResourcesSize > 0) {
243 for (const Resource &resource : orphanResources) {
244 resourceNames.append(resource.name());
245 }
246 inform(QStringLiteral("Found %1 orphan resources: %2").arg(orphanResourcesSize).arg(resourceNames.join(QLatin1Char(','))));
247 for (const QString &resourceName : std::as_const(resourceNames)) {
248 inform(QStringLiteral("Removing resource %1").arg(resourceName));
249 m_akonadi->resourceManager().removeResourceInstance(resourceName);
250 }
251 }
252}
253
254void StorageJanitor::findOrphanedCollections()
255{
256 SelectQueryBuilder<Collection> qb(m_dataStore.get());
257 qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
258 qb.addValueCondition(Resource::idFullColumnName(), Query::Is, QVariant());
259
260 if (!qb.exec()) {
261 inform("Failed to query orphaned collections, skipping test");
262 return;
263 }
264 const Collection::List orphans = qb.result();
265 if (!orphans.isEmpty()) {
266 inform(QLatin1StringView("Found ") + QString::number(orphans.size()) + QLatin1StringView(" orphan collections."));
267 // TODO: attach to lost+found resource
268 }
269}
270
271void StorageJanitor::checkCollectionTreeConsistency()
272{
273 const Collection::List cols = Collection::retrieveAll(m_dataStore.get());
274 std::for_each(cols.begin(), cols.end(), [this](const Collection &col) {
275 checkPathToRoot(col);
276 });
277}
278
279void StorageJanitor::checkPathToRoot(const Collection &col)
280{
281 if (col.parentId() == 0) {
282 return;
283 }
284 const Collection parent = col.parent(m_dataStore.get());
285 if (!parent.isValid()) {
286 inform(QLatin1StringView("Collection \"") + col.name() + QLatin1StringView("\" (id: ") + QString::number(col.id())
287 + QLatin1StringView(") has no valid parent."));
288 // TODO fix that by attaching to a top-level lost+found folder
289 return;
290 }
291
292 if (col.resourceId() != parent.resourceId()) {
293 inform(QLatin1StringView("Collection \"") + col.name() + QLatin1StringView("\" (id: ") + QString::number(col.id())
294 + QLatin1StringView(") belongs to a different resource than its parent."));
295 // can/should we actually fix that?
296 }
297
298 checkPathToRoot(parent);
299}
300
301void StorageJanitor::findOrphanedItems()
302{
303 SelectQueryBuilder<PimItem> qb(m_dataStore.get());
304 qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
305 qb.addValueCondition(Collection::idFullColumnName(), Query::Is, QVariant());
306 if (!qb.exec()) {
307 inform("Failed to query orphaned items, skipping test");
308 return;
309 }
310 const PimItem::List orphans = qb.result();
311 if (!orphans.isEmpty()) {
312 inform(QLatin1StringView("Found ") + QString::number(orphans.size()) + QLatin1StringView(" orphan items."));
313 // Attach to lost+found collection
314 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR ORPHANS"));
315 QueryBuilder qb(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Update);
316 qint64 col = lostAndFoundCollection();
317 if (col == -1) {
318 return;
319 }
320 qb.setColumnValue(PimItem::collectionIdColumn(), col);
321 qb.addValueCondition(PimItem::idFullColumnName(),
322 Query::In,
323 orphans | Views::transform([](const auto &item) {
324 return item.id();
325 }) | Actions::toQList);
326 if (qb.exec() && transaction.commit()) {
327 inform(QLatin1StringView("Moved orphan items to collection ") + QString::number(col));
328 } else {
329 inform(QLatin1StringView("Error moving orphan items to collection ") + QString::number(col) + QLatin1StringView(" : ")
330 + qb.query().lastError().text());
331 }
332 }
333}
334
335void StorageJanitor::findOrphanedParts()
336{
337 SelectQueryBuilder<Part> qb(m_dataStore.get());
338 qb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
339 qb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
340 if (!qb.exec()) {
341 inform("Failed to query orphaned parts, skipping test");
342 return;
343 }
344 const Part::List orphans = qb.result();
345 if (!orphans.isEmpty()) {
346 inform(QLatin1StringView("Found ") + QString::number(orphans.size()) + QLatin1StringView(" orphan parts."));
347 // TODO: create lost+found items for those? delete?
348 }
349}
350
351void StorageJanitor::findOrphanedPimItemFlags()
352{
353 QueryBuilder sqb(m_dataStore.get(), PimItemFlagRelation::tableName(), QueryBuilder::Select);
354 sqb.addColumn(PimItemFlagRelation::leftFullColumnName());
355 sqb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), PimItemFlagRelation::leftFullColumnName(), PimItem::idFullColumnName());
356 sqb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
357 if (!sqb.exec()) {
358 inform("Failed to query orphaned item flags, skipping test");
359 return;
360 }
361
363 while (sqb.query().next()) {
364 ids.append(sqb.query().value(0).toInt());
365 }
366 sqb.query().finish();
367 if (!ids.empty()) {
368 QueryBuilder qb(m_dataStore.get(), PimItemFlagRelation::tableName(), QueryBuilder::Delete);
369 qb.addValueCondition(PimItemFlagRelation::leftFullColumnName(), Query::In, ids);
370 if (!qb.exec()) {
371 qCCritical(AKONADISERVER_LOG) << "Error:" << qb.query().lastError().text();
372 return;
373 }
374
375 inform(QLatin1StringView("Found and deleted ") + QString::number(ids.size()) + QLatin1StringView(" orphan pim item flags."));
376 }
377}
378
379struct RelationDesc {
380 QString tableName;
381 QString deduplEntityIdColumnName;
382};
383
384template<typename DeduplEntity>
385std::optional<int> findDuplicatesImpl(DataStore *dataStore, const QString &nameColumn, const RelationDesc &relation)
386{
387 QueryBuilder sqb(dataStore, DeduplEntity::tableName(), QueryBuilder::Select);
388 sqb.addColumns({DeduplEntity::idColumn(), nameColumn});
389 sqb.addSortColumn(DeduplEntity::idColumn());
390 if (!sqb.exec()) {
391 return std::nullopt;
392 }
393
395 while (sqb.query().next()) {
396 const auto id = sqb.query().value(0).toLongLong();
397 const auto name = sqb.query().value(1).toString();
398
399 auto it = duplicates.find(name.trimmed());
400 if (it == duplicates.end()) {
401 it = duplicates.insert(name.trimmed(), QVariantList{});
402 }
403 it->push_back(id);
404 }
405
406 int removed = 0;
407 for (const auto &[duplicateName, duplicateIds] : duplicates.asKeyValueRange()) {
408 if (duplicateIds.size() <= 1) {
409 // Not duplicated
410 continue;
411 }
412
413 Transaction transaction(dataStore, QStringLiteral("StorageJanitor deduplicate %1 %2").arg(DeduplEntity::tableName(), duplicateName));
414
415 // Update all relations with duplicated entity to use the lowest entity ID, so we can remove the
416 // duplicates afterwards
417 const auto firstId = duplicateIds.takeFirst();
418
419 QueryBuilder updateQb(dataStore, relation.tableName, QueryBuilder::Update);
420 updateQb.setColumnValue(relation.deduplEntityIdColumnName, firstId);
421 updateQb.addValueCondition(relation.deduplEntityIdColumnName, Query::In, duplicateIds);
422 if (!updateQb.exec()) {
423 continue;
424 }
425
426 // Remove the duplicated entities
427 QueryBuilder removeQb(dataStore, DeduplEntity::tableName(), QueryBuilder::Delete);
428 removeQb.addValueCondition(DeduplEntity::idColumn(), Query::In, duplicateIds);
429 if (!removeQb.exec()) {
430 continue;
431 }
432
433 ++removed;
434
435 transaction.commit();
436 }
437
438 return removed;
439}
440
441void StorageJanitor::findDuplicateFlags()
442{
443 const auto removed =
444 findDuplicatesImpl<Flag>(m_dataStore.get(), Flag::nameFullColumnName(), {PimItemFlagRelation::tableName(), PimItemFlagRelation::rightFullColumnName()});
445 if (removed) {
446 inform(u"Removed " % QString::number(*removed) % u" duplicate item flags");
447 } else {
448 inform("Error while trying to remove duplicate Flags");
449 }
450}
451
452void StorageJanitor::findDuplicateMimeTypes()
453{
454 const auto removed =
455 findDuplicatesImpl<MimeType>(m_dataStore.get(), MimeType::nameFullColumnName(), {PimItem::tableName(), PimItem::mimeTypeIdFullColumnName()});
456 if (removed) {
457 inform(u"Removed " % QString::number(*removed) % u" duplicate mime types");
458 } else {
459 inform("Error while trying to remove duplicate MimeTypes");
460 }
461}
462
463void StorageJanitor::findDuplicatePartTypes()
464{
465 // Good thing that SQL is ANSI/ISO standardized...
467 if (DbType::type(m_dataStore->database()) == DbType::MySQL) {
468 nameColumn = QStringLiteral("CONCAT_WS(':', %1, %2) AS name");
469 } else {
470 nameColumn = QStringLiteral("(%1 || ':' || %2) AS name");
471 }
472
473 const auto removed = findDuplicatesImpl<PartType>(m_dataStore.get(),
474 nameColumn.arg(PartType::nsFullColumnName(), PartType::nameFullColumnName()),
475 {Part::tableName(), Part::partTypeIdFullColumnName()});
476 if (removed) {
477 inform(u"Removed " % QString::number(*removed) % u" duplicate part types");
478 } else {
479 inform("Error while trying to remove duplicate PartTypes");
480 }
481}
482
483void StorageJanitor::findDuplicateTagTypes()
484{
485 const auto removed = findDuplicatesImpl<TagType>(m_dataStore.get(), TagType::nameFullColumnName(), {Tag::tableName(), Tag::typeIdFullColumnName()});
486 if (removed) {
487 inform(u"Removed " % QString::number(*removed) % u" duplicate tag types");
488 } else {
489 inform("Error while trying to remove duplicate TagTypes");
490 }
491}
492
493void StorageJanitor::findDuplicateRelationTypes()
494{
495 const auto removed =
496 findDuplicatesImpl<RelationType>(m_dataStore.get(), RelationType::nameFullColumnName(), {Relation::tableName(), Relation::typeIdFullColumnName()});
497 if (removed) {
498 inform(u"Removed " % QString::number(*removed) % u" duplicate relation types");
499 } else {
500 inform("Error while trying to remove duplicate RelationTypes");
501 }
502}
503
504void StorageJanitor::findOverlappingParts()
505{
506 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
507 qb.addColumn(Part::dataColumn());
508 qb.addColumn(QLatin1StringView("count(") + Part::idColumn() + QLatin1StringView(") as cnt"));
509 qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
510 qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
511 qb.addGroupColumn(Part::dataColumn());
512 qb.addValueCondition(QLatin1StringView("count(") + Part::idColumn() + QLatin1StringView(")"), Query::Greater, 1, QueryBuilder::HavingCondition);
513 if (!qb.exec()) {
514 inform("Failed to query overlapping parts, skipping test");
515 return;
516 }
517
518 int count = 0;
519 while (qb.query().next()) {
520 ++count;
521 inform(QLatin1StringView("Found overlapping part data: ") + qb.query().value(0).toString());
522 // TODO: uh oh, this is bad, how do we recover from that?
523 }
524 qb.query().finish();
525
526 if (count > 0) {
527 inform(QLatin1StringView("Found ") + QString::number(count) + QLatin1StringView(" overlapping parts - bad."));
528 }
529}
530
531void StorageJanitor::verifyExternalParts()
532{
535
536 // list all files
537 const QString dataDir = StandardDirs::saveDir("data", QStringLiteral("file_db_data"));
539 while (it.hasNext()) {
540 existingFiles.insert(it.next());
541 }
542 existingFiles.remove(dataDir + QDir::separator() + QLatin1Char('.'));
543 existingFiles.remove(dataDir + QDir::separator() + QLatin1StringView(".."));
544 inform(QLatin1StringView("Found ") + QString::number(existingFiles.size()) + QLatin1StringView(" external files."));
545
546 // list all parts from the db which claim to have an associated file
547 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
548 qb.addColumn(Part::dataColumn());
549 qb.addColumn(Part::pimItemIdColumn());
550 qb.addColumn(Part::idColumn());
551 qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
552 qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
553 if (!qb.exec()) {
554 inform("Failed to query existing parts, skipping test");
555 return;
556 }
557 while (qb.query().next()) {
558 const auto filename = qb.query().value(0).toByteArray();
559 const auto pimItemId = qb.query().value(1).value<Entity::Id>();
560 const auto partId = qb.query().value(2).value<Entity::Id>();
562 if (!filename.isEmpty()) {
563 partPath = ExternalPartStorage::resolveAbsolutePath(filename);
564 } else {
565 partPath = ExternalPartStorage::resolveAbsolutePath(ExternalPartStorage::nameForPartId(partId));
566 }
567 if (existingFiles.contains(partPath)) {
568 usedFiles.insert(partPath);
569 } else {
570 inform(QLatin1StringView("Cleaning up missing external file: ") + partPath + QLatin1StringView(" for item: ") + QString::number(pimItemId)
571 + QLatin1StringView(" on part: ") + QString::number(partId));
572
573 Part part;
574 part.setId(partId);
575 part.setPimItemId(pimItemId);
576 part.setData(QByteArray());
577 part.setDatasize(0);
578 part.setStorage(Part::Internal);
579 part.update(m_dataStore.get());
580 }
581 }
582 qb.query().finish();
583 inform(QLatin1StringView("Found ") + QString::number(usedFiles.size()) + QLatin1StringView(" external parts."));
584
585 // see what's left and move it to lost+found
587 if (!unreferencedFiles.isEmpty()) {
588 const QString lfDir = StandardDirs::saveDir("data", QStringLiteral("file_lost+found"));
589 for (const QString &file : unreferencedFiles) {
590 inform(QLatin1StringView("Found unreferenced external file: ") + file);
591 const QFileInfo f(file);
592 QFile::rename(file, lfDir + QDir::separator() + f.fileName());
593 }
594 inform(QStringLiteral("Moved %1 unreferenced files to lost+found.").arg(unreferencedFiles.size()));
595 } else {
596 inform("Found no unreferenced external files.");
597 }
598}
599
600void StorageJanitor::findDirtyObjects()
601{
602 SelectQueryBuilder<Collection> cqb(m_dataStore.get());
603 cqb.setSubQueryMode(Query::Or);
604 cqb.addValueCondition(Collection::remoteIdColumn(), Query::Is, QVariant());
605 cqb.addValueCondition(Collection::remoteIdColumn(), Query::Equals, QString());
606 if (!cqb.exec()) {
607 inform("Failed to query collections without RID, skipping test");
608 return;
609 }
610 const Collection::List ridLessCols = cqb.result();
611 for (const Collection &col : ridLessCols) {
612 inform(QLatin1StringView("Collection \"") + col.name() + QLatin1StringView("\" (id: ") + QString::number(col.id())
613 + QLatin1StringView(") has no RID."));
614 }
615 inform(QLatin1StringView("Found ") + QString::number(ridLessCols.size()) + QLatin1StringView(" collections without RID."));
616
617 SelectQueryBuilder<PimItem> iqb1(m_dataStore.get());
618 iqb1.setSubQueryMode(Query::Or);
619 iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Is, QVariant());
620 iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, QString());
621 if (!iqb1.exec()) {
622 inform("Failed to query items without RID, skipping test");
623 return;
624 }
625 const PimItem::List ridLessItems = iqb1.result();
626 for (const PimItem &item : ridLessItems) {
627 inform(QLatin1StringView("Item \"") + QString::number(item.id()) + QLatin1StringView("\" in collection \"") + QString::number(item.collectionId())
628 + QLatin1StringView("\" has no RID."));
629 }
630 inform(QLatin1StringView("Found ") + QString::number(ridLessItems.size()) + QLatin1StringView(" items without RID."));
631
632 SelectQueryBuilder<PimItem> iqb2(m_dataStore.get());
633 iqb2.addValueCondition(PimItem::dirtyColumn(), Query::Equals, true);
634 iqb2.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
635 iqb2.addSortColumn(PimItem::idFullColumnName());
636 if (!iqb2.exec()) {
637 inform("Failed to query dirty items, skipping test");
638 return;
639 }
640 const PimItem::List dirtyItems = iqb2.result();
641 for (const PimItem &item : dirtyItems) {
642 inform(QLatin1StringView("Item \"") + QString::number(item.id()) + QLatin1StringView("\" has RID and is dirty."));
643 }
644 inform(QLatin1StringView("Found ") + QString::number(dirtyItems.size()) + QLatin1StringView(" dirty items."));
645}
646
647void StorageJanitor::findRIDDuplicates()
648{
649 QueryBuilder qb(m_dataStore.get(), Collection::tableName(), QueryBuilder::Select);
650 qb.addColumn(Collection::idColumn());
651 qb.addColumn(Collection::nameColumn());
652 qb.exec();
653
654 while (qb.query().next()) {
655 const auto colId = qb.query().value(0).value<Collection::Id>();
656 const QString name = qb.query().value(1).toString();
657 inform(QStringLiteral("Checking ") + name);
658
659 QueryBuilder duplicates(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Select);
660 duplicates.addColumn(PimItem::remoteIdColumn());
661 duplicates.addColumn(QStringLiteral("count(") + PimItem::idColumn() + QStringLiteral(") as cnt"));
662 duplicates.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
663 duplicates.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
664 duplicates.addGroupColumn(PimItem::remoteIdColumn());
665 duplicates.addValueCondition(QStringLiteral("count(") + PimItem::idColumn() + QLatin1Char(')'), Query::Greater, 1, QueryBuilder::HavingCondition);
666 duplicates.exec();
667
668 Akonadi::Server::Collection col = Akonadi::Server::Collection::retrieveById(m_dataStore.get(), colId);
669 const QList<Akonadi::Server::MimeType> contentMimeTypes = col.mimeTypes(m_dataStore.get());
670 QVariantList contentMimeTypesVariantList;
671 contentMimeTypesVariantList.reserve(contentMimeTypes.count());
672 for (const Akonadi::Server::MimeType &mimeType : contentMimeTypes) {
674 }
675 while (duplicates.query().next()) {
676 const QString rid = duplicates.query().value(0).toString();
677
678 Query::Condition condition(Query::And);
679 condition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, rid);
680 condition.addValueCondition(PimItem::mimeTypeIdColumn(), Query::NotIn, contentMimeTypesVariantList);
681 condition.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
682
683 QueryBuilder items(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Select);
684 items.addColumn(PimItem::idColumn());
685 items.addCondition(condition);
686 if (!items.exec()) {
687 inform(QStringLiteral("Error while deleting duplicates: ") + items.query().lastError().text());
688 continue;
689 }
690 QVariantList itemsIds;
691 while (items.query().next()) {
692 itemsIds.push_back(items.query().value(0));
693 }
694 items.query().finish();
695 if (itemsIds.isEmpty()) {
696 // the mimetype filter may have dropped some entries from the
697 // duplicates query
698 continue;
699 }
700
701 inform(QStringLiteral("Found duplicates ") + rid);
702
703 SelectQueryBuilder<Part> parts(m_dataStore.get());
704 parts.addValueCondition(Part::pimItemIdFullColumnName(), Query::In, QVariant::fromValue(itemsIds));
705 parts.addValueCondition(Part::storageFullColumnName(), Query::Equals, static_cast<int>(Part::External));
706 if (parts.exec()) {
707 const auto partsList = parts.result();
708 for (const auto &part : partsList) {
709 bool exists = false;
710 const auto filename = ExternalPartStorage::resolveAbsolutePath(part.data(), &exists);
711 if (exists) {
712 QFile::remove(filename);
713 }
714 }
715 }
716
717 items = QueryBuilder(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Delete);
718 items.addCondition(condition);
719 if (!items.exec()) {
720 inform(QStringLiteral("Error while deleting duplicates ") + items.query().lastError().text());
721 }
722 }
723 duplicates.query().finish();
724 }
725 qb.query().finish();
726}
727
729{
730 const DbType::Type dbType = DbType::type(m_dataStore->database());
731 if (dbType == DbType::MySQL || dbType == DbType::PostgreSQL) {
732 inform("vacuuming database, that'll take some time and require a lot of temporary disk space...");
733 const auto tables = allDatabaseTables();
734 for (const QString &table : tables) {
735 inform(QStringLiteral("optimizing table %1...").arg(table));
736
738 if (dbType == DbType::MySQL) {
739 queryStr = QLatin1StringView("OPTIMIZE TABLE ") + table;
740 } else if (dbType == DbType::PostgreSQL) {
741 queryStr = QLatin1StringView("VACUUM FULL ANALYZE ") + table;
742 } else {
743 continue;
744 }
745 QSqlQuery q(m_dataStore->database());
746 if (!q.exec(queryStr)) {
747 qCCritical(AKONADISERVER_LOG) << "failed to optimize table" << table << ":" << q.lastError().text();
748 }
749 }
750 inform("vacuum done");
751 } else {
752 inform("Vacuum not supported for this database backend. (Sqlite backend)");
753 }
754
755 Q_EMIT done();
756}
757
758void StorageJanitor::checkSizeTreshold()
759{
760 {
761 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
762 qb.addColumn(Part::idFullColumnName());
763 qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::Internal);
764 qb.addValueCondition(Part::datasizeFullColumnName(), Query::Greater, m_dbConfig->sizeThreshold());
765 if (!qb.exec()) {
766 inform("Failed to query parts larger than threshold, skipping test");
767 return;
768 }
769
770 QSqlQuery query = qb.query();
771 inform(QStringLiteral("Found %1 parts to be moved to external files").arg(query.size()));
772
773 while (query.next()) {
774 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD"));
775 Part part = Part::retrieveById(m_dataStore.get(), query.value(0).toLongLong());
776 const QByteArray name = ExternalPartStorage::nameForPartId(part.id());
777 const QString partPath = ExternalPartStorage::resolveAbsolutePath(name);
779 if (f.exists()) {
780 qCDebug(AKONADISERVER_LOG) << "External payload file" << name << "already exists";
781 // That however is not a critical issue, since the part is not external,
782 // so we can safely overwrite it
783 }
785 qCCritical(AKONADISERVER_LOG) << "Failed to open file" << name << "for writing";
786 continue;
787 }
788 if (f.write(part.data()) != part.datasize()) {
789 qCCritical(AKONADISERVER_LOG) << "Failed to write data to payload file" << name;
790 f.remove();
791 continue;
792 }
793
794 part.setData(name);
795 part.setStorage(Part::External);
796 if (!part.update(m_dataStore.get()) || !transaction.commit()) {
797 qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
798 f.remove();
799 continue;
800 }
801
802 inform(QStringLiteral("Moved part %1 from database into external file %2").arg(part.id()).arg(QString::fromLatin1(name)));
803 }
804 query.finish();
805 }
806
807 {
808 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
809 qb.addColumn(Part::idFullColumnName());
810 qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
811 qb.addValueCondition(Part::datasizeFullColumnName(), Query::Less, DbConfig::configuredDatabase()->sizeThreshold());
812 if (!qb.exec()) {
813 inform("Failed to query parts smaller than threshold, skipping test");
814 return;
815 }
816
817 QSqlQuery query = qb.query();
818 inform(QStringLiteral("Found %1 parts to be moved to database").arg(query.size()));
819
820 while (query.next()) {
821 Transaction transaction(m_dataStore.get(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD 2"));
822 Part part = Part::retrieveById(m_dataStore.get(), query.value(0).toLongLong());
823 const QString partPath = ExternalPartStorage::resolveAbsolutePath(part.data());
825 if (!f.exists()) {
826 qCCritical(AKONADISERVER_LOG) << "Part file" << part.data() << "does not exist";
827 continue;
828 }
829 if (!f.open(QIODevice::ReadOnly)) {
830 qCCritical(AKONADISERVER_LOG) << "Failed to open part file" << part.data() << "for reading";
831 continue;
832 }
833
834 part.setStorage(Part::Internal);
835 part.setData(f.readAll());
836 if (part.data().size() != part.datasize()) {
837 qCCritical(AKONADISERVER_LOG) << "Sizes of" << part.id() << "data don't match";
838 continue;
839 }
840 if (!part.update(m_dataStore.get()) || !transaction.commit()) {
841 qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
842 continue;
843 }
844
845 f.close();
846 f.remove();
847 inform(QStringLiteral("Moved part %1 from external file into database").arg(part.id()));
848 }
849 query.finish();
850 }
851}
852
853void StorageJanitor::migrateToLevelledCacheHierarchy()
854{
855 /// First, check whether that's still necessary
856 const QString db_data = StandardDirs::saveDir("data", QStringLiteral("file_db_data"));
858 if (entries.isEmpty()) {
859 inform("No external parts in legacy location, skipping migration");
860 return;
861 }
862
863 QueryBuilder qb(m_dataStore.get(), Part::tableName(), QueryBuilder::Select);
864 qb.addColumn(Part::idColumn());
865 qb.addColumn(Part::dataColumn());
866 qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
867 if (!qb.exec()) {
868 inform("Failed to query external payload parts, skipping test");
869 return;
870 }
871
872 QSqlQuery query = qb.query();
873 while (query.next()) {
874 const qint64 id = query.value(0).toLongLong();
875 const QByteArray data = query.value(1).toByteArray();
876 const QString fileName = QString::fromUtf8(data);
877 bool oldExists = false;
878 bool newExists = false;
879 // Resolve the current path
880 const QString currentPath = ExternalPartStorage::resolveAbsolutePath(fileName, &oldExists);
881 // Resolve the new path with legacy fallback disabled, so that it always
882 // returns the new levelled-cache path, even when the old one exists
883 const QString newPath = ExternalPartStorage::resolveAbsolutePath(fileName, &newExists, false);
884 if (!oldExists) {
885 qCCritical(AKONADISERVER_LOG) << "Old payload part does not exist, skipping part" << fileName;
886 continue;
887 }
888 if (currentPath != newPath) {
889 if (newExists) {
890 qCCritical(AKONADISERVER_LOG) << "Part is in legacy location, but the destination file already exists, skipping part" << fileName;
891 continue;
892 }
893
894 QFile f(currentPath);
895 if (!f.rename(newPath)) {
896 qCCritical(AKONADISERVER_LOG) << "Failed to move part from" << currentPath << " to " << newPath << ":" << f.errorString();
897 continue;
898 }
899 inform(QStringLiteral("Migrated part %1 to new levelled cache").arg(id));
900 }
901 }
902 query.finish();
903}
904
905void StorageJanitor::findOrphanSearchIndexEntries()
906{
907 QueryBuilder qb(m_dataStore.get(), Collection::tableName(), QueryBuilder::Select);
908 qb.addSortColumn(Collection::idColumn(), Query::Ascending);
909 qb.addColumn(Collection::idColumn());
910 qb.addColumn(Collection::isVirtualColumn());
911 if (!qb.exec()) {
912 inform("Failed to query collections, skipping test");
913 return;
914 }
915
916 QDBusInterface iface(DBus::agentServiceName(QStringLiteral("akonadi_indexing_agent"), DBus::Agent),
917 QStringLiteral("/"),
918 QStringLiteral("org.freedesktop.Akonadi.Indexer"),
920 if (!iface.isValid()) {
921 inform("Akonadi Indexing Agent is not running, skipping test");
922 return;
923 }
924
925 QSqlQuery query = qb.query();
926 while (query.next()) {
927 const qint64 colId = query.value(0).toLongLong();
928 // Skip virtual collections, they are not indexed
929 if (query.value(1).toBool()) {
930 inform(QStringLiteral("Skipping virtual Collection %1").arg(colId));
931 continue;
932 }
933
934 inform(QStringLiteral("Checking Collection %1 search index...").arg(colId));
935 SearchRequest req("StorageJanitor", m_akonadi->searchManager(), m_akonadi->agentSearchManager());
936 req.setStoreResults(true);
937 req.setCollections({colId});
938 req.setRemoteSearch(false);
939 req.setQuery(QStringLiteral("{ }")); // empty query to match all
942 col.setId(colId);
943 const auto colMts = col.mimeTypes();
944 if (colMts.isEmpty()) {
945 // No mimetypes means we don't know which search store to look into,
946 // skip it.
947 continue;
948 }
949 mts.reserve(colMts.count());
950 for (const auto &mt : colMts) {
951 mts << mt.name();
952 }
953 req.setMimeTypes(mts);
954 req.exec();
955 auto searchResults = req.results();
956
957 QueryBuilder iqb(m_dataStore.get(), PimItem::tableName(), QueryBuilder::Select);
958 iqb.addColumn(PimItem::idColumn());
959 iqb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
960 if (!iqb.exec()) {
961 inform(QStringLiteral("Failed to query items in collection %1").arg(colId));
962 continue;
963 }
964
965 QSqlQuery itemQuery = iqb.query();
966 while (itemQuery.next()) {
967 searchResults.remove(itemQuery.value(0).toLongLong());
968 }
969 itemQuery.finish();
970
971 if (!searchResults.isEmpty()) {
972 inform(QStringLiteral("Collection %1 search index contains %2 orphan items. Scheduling reindexing").arg(colId).arg(searchResults.count()));
973 iface.call(QDBus::NoBlock, QStringLiteral("reindexCollection"), colId);
974 }
975 }
976 query.finish();
977}
978
979void StorageJanitor::ensureSearchCollection()
980{
981 static const auto searchResourceName = QStringLiteral("akonadi_search_resource");
982
984 if (!searchResource.isValid()) {
986 searchResource.setIsVirtual(true);
987 if (!searchResource.insert(m_dataStore.get())) {
988 inform(QStringLiteral("Failed to create Search resource."));
989 return;
990 }
991 }
992
993 auto searchCols = Collection::retrieveFiltered(m_dataStore.get(), Collection::resourceIdColumn(), searchResource.id());
994 if (searchCols.isEmpty()) {
996 searchCol.setId(1);
997 searchCol.setName(QStringLiteral("Search"));
998 searchCol.setResource(searchResource);
999 searchCol.setIndexPref(Collection::False);
1000 searchCol.setIsVirtual(true);
1001 if (!searchCol.insert(m_dataStore.get())) {
1002 inform(QStringLiteral("Failed to create Search Collection"));
1003 return;
1004 }
1005 }
1006}
1007
1008void StorageJanitor::expireCollectionStatisticsCache()
1009{
1010 m_akonadi->collectionStatistics().expireCache();
1011}
1012
1013void StorageJanitor::inform(const char *msg)
1014{
1015 inform(QLatin1StringView(msg));
1016}
1017
1018void StorageJanitor::inform(const QString &msg)
1019{
1020 qCDebug(AKONADISERVER_LOG) << msg;
1021 Q_EMIT information(msg);
1022}
1023
1024#include "moc_storagejanitor.cpp"
Represents a collection of PIM items.
Definition collection.h:62
qint64 Id
Describes the unique id type.
Definition collection.h:79
This class handles all the database access.
Definition datastore.h:95
A base class that provides an unique access layer to configuration and initialization of different da...
Definition dbconfig.h:21
virtual qint64 sizeThreshold() const
Payload data bigger than this value will be stored in separate files, instead of the database.
Definition dbconfig.cpp:138
static DbConfig * configuredDatabase()
Returns the DbConfig instance for the database the user has configured.
Definition dbconfig.cpp:77
Helper class to construct arbitrary SQL queries.
@ HavingCondition
add condition to HAVING part of the query NOTE: only supported for SELECT queries
@ LeftJoin
NOTE: only supported for SELECT queries.
Represents a WHERE condition tree.
Definition query.h:62
Helper class for creating and executing database SELECT queries.
Q_SCRIPTABLE Q_NOREPLY void vacuum()
Triggers a vacuuming of the database, that is compacting of unused space.
StorageJanitor(AkonadiServer *mAkonadi, DbConfig *config=DbConfig::configuredDatabase())
Use AkThread::create() to create and start a new StorageJanitor thread.
Q_SCRIPTABLE Q_NOREPLY void check()
Triggers a consistency check of the internal storage.
Q_SCRIPTABLE void information(const QString &msg)
Sends informational messages to a possible UI for this.
Helper class for DataStore transaction handling.
Definition transaction.h:23
Type
Supported database types.
Definition dbtype.h:19
Type type(const QSqlDatabase &db)
Returns the type of the given database object.
Definition dbtype.cpp:11
Helper integration between Akonadi and Qt.
char * toString(const EngineQuery &query)
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
KCALUTILS_EXPORT QString mimeType()
QString name(StandardAction id)
A glue between Qt and the standard library.
QDateTime currentDateTime()
QDBusConnection sessionBus()
QFileInfoList entryInfoList(Filters filters, SortFlags sort) const const
QChar separator()
bool remove()
bool rename(const QString &newName)
void append(QList< T > &&value)
qsizetype count() const const
bool empty() const const
qsizetype size() const const
T value(qsizetype i) const const
iterator end()
iterator find(const Key &key)
iterator insert(const Key &key, const T &value)
T value(const Key &key, const T &defaultValue) const const
Q_EMITQ_EMIT
const QObjectList & children() const const
QObject * parent() const const
T qobject_cast(QObject *object)
QString text() const const
bool exec()
QSqlError lastError() const const
QString fromLatin1(QByteArrayView str)
QString fromUtf8(QByteArrayView str)
QString number(double n, char format, int precision)
QString trimmed() const const
QVariant fromValue(T &&value)
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Fri Jun 21 2024 11:57:49 by doxygen 1.10.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.