9#include "ControlManager.h"
10#include "akonadidbmigrator_debug.h"
11#include "akonadifull-version.h"
12#include "akonadischema.h"
15#include "private/dbus_p.h"
16#include "private/standarddirs_p.h"
17#include "storage/countquerybuilder.h"
18#include "storage/datastore.h"
19#include "storage/dbconfig.h"
20#include "storage/dbconfigmysql.h"
21#include "storage/dbconfigpostgresql.h"
22#include "storage/dbconfigsqlite.h"
23#include "storage/dbintrospector.h"
24#include "storage/querybuilder.h"
25#include "storage/schematypes.h"
26#include "storage/transaction.h"
27#include "storagejanitor.h"
30#include <QCommandLineOption>
31#include <QCommandLineParser>
32#include <QCoreApplication>
33#include <QDBusConnection>
35#include <QDBusInterface>
37#include <QElapsedTimer>
42#include <QSqlDatabase>
48#include <KLocalizedString>
53using namespace Akonadi::Server;
54using namespace AkRanges;
55using namespace std::chrono_literals;
57Q_DECLARE_METATYPE(UIDelegate::Result);
62constexpr size_t maxTransactionSize = 1000;
64class MigratorDataStoreFactory :
public DataStoreFactory
67 explicit MigratorDataStoreFactory(DbConfig *config)
72 DataStore *createStore()
override
74 class MigratorDataStore :
public DataStore
77 explicit MigratorDataStore(DbConfig *config)
82 return new MigratorDataStore(m_dbConfig);
86 DbConfig *
const m_dbConfig;
91 Q_DISABLE_COPY_MOVE(Rollback)
107 for (
auto it = mRollbacks.rbegin(); it != mRollbacks.rend(); ++it) {
114 void add(T &&rollback)
116 mRollbacks.push_back(std::forward<T>(rollback));
120 std::vector<std::function<void()>> mRollbacks;
123void stopDatabaseServer(
DbConfig *config)
128bool createDatabase(
DbConfig *config)
134 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Invalid database object during initial database connection";
138 const auto closeDb = qScopeGuard([&db]() {
148 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Failed to use database" << config->
databaseName();
149 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Database error:" << db.lastError().text();
150 qCDebug(AKONADIDBMIGRATOR_LOG) <<
"Trying to create database now...";
157 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Failed to connect to database!";
158 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Database error:" << db.lastError().text();
164 if (!
query.exec(QStringLiteral(
"CREATE DATABASE %1").arg(config->
databaseName()))) {
165 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Failed to create database";
166 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Query error:" <<
query.lastError().text();
167 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Database error:" << db.lastError().text();
173std::unique_ptr<DataStore> prepareDatabase(
DbConfig *config)
177 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Failed to start the database server";
181 if (!createDatabase(config)) {
188 auto factory = std::make_unique<MigratorDataStoreFactory>(config);
189 std::unique_ptr<DataStore> store{factory->createStore()};
190 if (!store->database().isOpen()) {
193 if (!store->init()) {
204 stopDatabaseServer(config);
209 const auto idCol = std::find_if(table.columns.begin(), table.columns.end(), [](
const auto &col) {
210 return col.isPrimaryKey;
212 if (idCol == table.columns.end()) {
216 const auto getAutoIncrementValue = [](
DataStore *store,
const QString &table,
const QString &idCol) -> std::optional<qint64> {
221 if (!
query.exec(introspector->getAutoIncrementValueQuery(table, idCol))) {
222 qCCritical(AKONADIDBMIGRATOR_LOG) <<
query.lastError().text();
234 const auto setAutoIncrementValue = [](
DataStore *store,
const QString &table,
const QString &idCol, qint64 seq) ->
bool {
239 if (!
query.exec(introspector->updateAutoIncrementValueQuery(table, idCol, seq))) {
240 qCritical(AKONADIDBMIGRATOR_LOG) <<
query.lastError().text();
247 const auto seq = getAutoIncrementValue(sourceStore, table.name, idCol->name);
252 return setAutoIncrementValue(destStore, table.name, idCol->name, *seq);
261 case DbType::PostgreSQL:
267 case DbType::Unknown:
268 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Unknown database type";
273 if (!
query.exec(queryStr)) {
274 qCCritical(AKONADIDBMIGRATOR_LOG) <<
query.lastError().text();
283 const auto origFileName = StandardDirs::serverConfigFile(StandardDirs::ReadWrite);
284 const auto newFileName = QStringLiteral(
"%1.new").arg(origFileName);
287 settings.setValue(QStringLiteral(
"General/Driver"), targetEngine);
294 const auto enginelc = engine.
toLower();
296 return QStringLiteral(
"QSQLITE");
299 return QStringLiteral(
"QMYSQL");
302 return QStringLiteral(
"QPSQL");
305 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Invalid engine:" << engine;
309std::unique_ptr<DbConfig> dbConfigFromServerRc(
const QString &configFile,
bool overrideDbPath =
false)
312 const auto driver = settings.value(QStringLiteral(
"General/Driver")).toString();
313 std::unique_ptr<DbConfig> config;
316 config = std::make_unique<DbConfigSqlite>(configFile);
318 config = std::make_unique<DbConfigMysql>(configFile);
319 dbPathSuffix = QStringLiteral(
"/db_data");
321 config = std::make_unique<DbConfigPostgresql>(configFile);
322 dbPathSuffix = QStringLiteral(
"/db_data");
324 qCCritical(AKONADIDBMIGRATOR_LOG) <<
"Invalid driver: " << driver;
328 const auto dbPath = overrideDbPath ? StandardDirs::saveDir(
"data", QStringLiteral(
"db_migration%1").arg(dbPathSuffix)) :
QString{};
329 config->init(settings,
true, dbPath);
334bool isValidDbPath(
const QString &path)
341 return fi.exists() && (fi.isFile() || fi.isDir());
344std::error_code restoreDatabaseFromBackup(
const QString &backupPath,
const QString &originalPath)
349 std::filesystem::remove_all(originalPath.
toStdString(), ec);
358bool akonadiIsRunning()
361 return sessionIface->
isServiceRegistered(DBus::serviceName(DBus::ControlLock)) || sessionIface->isServiceRegistered(DBus::serviceName(DBus::Server));
366 static constexpr auto shutdownTimeout = 5s;
368 org::freedesktop::Akonadi::ControlManager manager(DBus::serviceName(DBus::Control), QStringLiteral(
"/ControlManager"),
QDBusConnection::sessionBus());
369 if (!manager.isValid()) {
377 while (akonadiIsRunning() && timer.
durationElapsed() <= shutdownTimeout) {
381 return timer.
durationElapsed() <= shutdownTimeout && !akonadiIsRunning();
390bool acquireAkonadiLock()
393 auto reply = connIface->
registerService(DBus::serviceName(DBus::ControlLock));
398 reply = connIface->registerService(DBus::serviceName(DBus::UpgradeIndicator));
406bool releaseAkonadiLock()
410 connIface->unregisterService(DBus::serviceName(DBus::UpgradeIndicator));
416DbMigrator::DbMigrator(
const QString &targetEngine, UIDelegate *delegate,
QObject *parent)
418 , m_targetEngine(targetEngine)
419 , m_uiDelegate(delegate)
421 qRegisterMetaType<UIDelegate::Result>();
424DbMigrator::~DbMigrator()
431void DbMigrator::startMigration()
434 bool restartAkonadi =
false;
435 if (akonadiIsRunning()) {
436 emitInfo(
i18nc(
"@info:status",
"Stopping Akonadi service..."));
437 restartAkonadi =
true;
438 if (!stopAkonadi()) {
439 emitError(
i18nc(
"@info:status",
"Error: timeout while waiting for Akonadi to stop."));
440 emitCompletion(
false);
445 if (!acquireAkonadiLock()) {
446 emitError(
i18nc(
"@info:status",
"Error: couldn't acquire DBus lock for Akonadi."));
447 emitCompletion(
false);
451 const bool result = runMigrationThread();
453 releaseAkonadiLock();
454 if (restartAkonadi) {
455 emitInfo(
i18nc(
"@info:status",
"Starting Akonadi service..."));
459 emitCompletion(result);
464bool DbMigrator::runStorageJanitor(
DbConfig *dbConfig)
466 StorageJanitor janitor(dbConfig);
467 connect(&janitor, &StorageJanitor::done,
this, [
this]() {
468 emitInfo(
i18nc(
"@info:status",
"Database fsck completed"));
476bool DbMigrator::runMigrationThread()
480 const auto driver = driverFromEngineName(m_targetEngine);
481 if (driver.isEmpty()) {
482 emitError(
i18nc(
"@info:status",
"Invalid database engine \"%1\" - valid values are \"sqlite\", \"mysql\" and \"postgres\".", m_targetEngine));
487 const auto sourceServerCfgFile = backupAkonadiServerRc();
488 if (!sourceServerCfgFile) {
491 rollback.add([file = *sourceServerCfgFile]() {
496 const QString destServerCfgFile = createTmpAkonadiServerRc(driver);
497 rollback.add([destServerCfgFile]() {
502 auto sourceConfig = dbConfigFromServerRc(*sourceServerCfgFile);
504 emitError(
i18nc(
"@info:shell",
"Error: failed to configure source database server."));
508 if (sourceConfig->driverName() == driver) {
509 emitError(
i18nc(
"@info:shell",
"Source and destination database engines are the same."));
514 const auto sourceDbPath = sourceConfig->databasePath();
515 if (!isValidDbPath(sourceDbPath)) {
516 emitError(
i18nc(
"@info:shell",
"Error: failed to obtain path to source database data file or directory."));
521 auto destConfig = dbConfigFromServerRc(destServerCfgFile,
true);
523 emitError(
i18nc(
"@info:shell",
"Error: failed to configure the new database server."));
527 auto sourceStore = prepareDatabase(sourceConfig.get());
529 emitError(
i18nc(
"@info:shell",
"Error: failed to open existing database to migrate data from."));
532 auto destStore = prepareDatabase(destConfig.get());
534 emitError(
i18nc(
"@info:shell",
"Error: failed to open new database to migrate data to."));
539 emitInfo(
i18nc(
"@info:status",
"Running fsck on the source database"));
540 runStorageJanitor(sourceConfig.get());
542 const bool migrationSuccess = migrateTables(sourceStore.get(), destStore.get(), destConfig.get());
545 cleanupDatabase(sourceStore.get(), sourceConfig.get());
546 cleanupDatabase(destStore.get(), destConfig.get());
548 if (!migrationSuccess) {
553 rollback.add([
this, dbPath = destConfig->databasePath()]() {
555 std::filesystem::remove_all(dbPath.toStdString(), ec);
557 emitError(i18nc(
"@info:status %2 is error message",
558 "Error: failed to remove temporary database directory %1: %2",
560 QString::fromStdString(ec.message())));
565 emitInfo(
i18nc(
"@info:shell",
"Backing up original database..."));
566 const auto backupPath = moveDatabaseToBackupLocation(sourceConfig.get());
567 if (!backupPath.has_value()) {
572 rollback.add([
this, backupPath = *backupPath, sourceDbPath]() {
573 emitInfo(
i18nc(
"@info:status",
"Restoring database from backup %1 to %2", backupPath, sourceDbPath));
574 if (
const auto ec = restoreDatabaseFromBackup(backupPath, sourceDbPath); ec) {
575 emitError(
i18nc(
"@info:status %1 is error message",
"Error: failed to restore database from backup: %1",
QString::fromStdString(ec.message())));
581 if (!moveDatabaseToMainLocation(destConfig.get(), destServerCfgFile)) {
598 AkonadiSchema schema;
599 const int totalTables = schema.tables().size() + schema.relations().size();
603 for (
const auto &table : schema.tables()) {
605 emitProgress(table.name, doneTables, totalTables);
606 if (!copyTable(sourceStore, destStore, table)) {
607 emitError(
i18nc(
"@info:shell",
"Error has occurred while migrating table %1", table.name));
613 for (
const auto &relation : schema.relations()) {
614 const RelationTableDescription table{relation};
616 emitProgress(table.name, doneTables, totalTables);
617 if (!copyTable(sourceStore, destStore, table)) {
618 emitError(
i18nc(
"@info:shell",
"Error has occurred while migrating table %1", table.name));
631std::optional<QString> DbMigrator::moveDatabaseToBackupLocation(
DbConfig *config)
635 QDir backupDir = StandardDirs::saveDir(
"data", QStringLiteral(
"migration_backup"));
637 const auto result = questionYesNoSkip(
i18nc(
"@label",
"Backup directory already exists. Do you want to overwrite the previous backup?"));
638 if (result == UIDelegate::Result::Skip) {
641 if (result == UIDelegate::Result::No) {
642 emitError(
i18nc(
"@info:shell",
"Cannot proceed without backup. Migration interrupted."));
647 emitError(
i18nc(
"@info:shell",
"Failed to remove previous backup directory."));
652 backupDir.
mkpath(QStringLiteral(
"."));
658 std::filesystem::rename(dbPath, backupPath / *(--dbPath.end()), ec);
667std::optional<QString> DbMigrator::backupAkonadiServerRc()
669 const auto origFileName = StandardDirs::serverConfigFile(StandardDirs::ReadWrite);
670 const auto bkpFileName = QStringLiteral(
"%1.bkp").arg(origFileName);
674 const auto result = questionYesNo(
i18nc(
"@label",
"Backup file %1 already exists. Overwrite?", bkpFileName));
675 if (result != UIDelegate::Result::Yes) {
676 emitError(
i18nc(
"@info:status",
"Cannot proceed without backup. Migration interrupted."));
680 std::filesystem::remove(bkpFileName.toStdString(), ec);
682 emitError(
i18nc(
"@info:status",
"Error: Failed to remove existing backup file %1: %2", bkpFileName,
QString::fromStdString(ec.message())));
687 std::filesystem::copy(origFileName.toStdString(), bkpFileName.toStdString(), ec);
696bool DbMigrator::moveDatabaseToMainLocation(
DbConfig *destConfig,
const QString &destServerCfgFile)
702 const auto dbDestPath = dbSrcPath.parent_path().parent_path() / *(--dbSrcPath.end());
703 std::filesystem::rename(dbSrcPath, dbDestPath, ec);
705 emitError(
i18nc(
"@info:status %1 is error message",
706 "Error: failed to move migrated database to the primary location: %1",
719 std::filesystem::remove(dbSrcPath.parent_path(), ec);
723 std::filesystem::remove(StandardDirs::serverConfigFile(StandardDirs::ReadWrite).toStdString(), ec);
725 emitError(
i18nc(
"@info:status %1 is error message",
"Error: failed to remove original akonadiserverrc: %1",
QString::fromStdString(ec.message())));
728 std::filesystem::rename(destServerCfgFile.
toStdString(), StandardDirs::serverConfigFile(StandardDirs::ReadWrite).toStdString(), ec);
730 emitError(
i18nc(
"@info:status %1 is error message",
731 "Error: failed to move new akonadiserverrc to the primary location: %1",
741 const auto columns = table.columns | Views::transform([](
const auto &tbl) {
747 const auto totalRows = [](DataStore *store,
const QString &table) {
748 CountQueryBuilder countQb(store, table);
750 return countQb.result();
751 }(sourceStore, table.name);
754 QueryBuilder sourceQb(sourceStore, table.name);
755 sourceQb.addColumns(columns);
757 auto &sourceQuery = sourceQb.query();
761 QueryBuilder clearQb(destStore, table.name, QueryBuilder::Delete);
766 Transaction transaction(destStore, QStringLiteral(
"Migrator"));
768 size_t processed = 0;
771 while (sourceQuery.next()) {
773 QueryBuilder destQb(destStore, table.name, QueryBuilder::Insert);
774 destQb.setIdentificationColumn({});
775 for (
int col = 0; col < table.columns.size(); ++col) {
777 if (table.columns[col].type == QLatin1StringView(
"QDateTime")) {
778 value = sourceQuery.
value(col).toDateTime();
779 }
else if (table.columns[col].type == QLatin1StringView(
"bool")) {
780 value = sourceQuery.
value(col).toBool();
781 }
else if (table.columns[col].type == QLatin1StringView(
"QByteArray")) {
782 value = Utils::variantToByteArray(sourceQuery.value(col));
783 }
else if (table.columns[col].type == QLatin1StringView(
"QString")) {
784 value = Utils::variantToString(sourceQuery.value(col));
786 value = sourceQuery.
value(col);
788 destQb.setColumnValue(table.columns[col].name, value);
790 if (!destQb.exec()) {
791 qCWarning(AKONADIDBMIGRATOR_LOG) <<
"Failed to insert row into table" << table.name <<
":" << destQb.query().lastError().text();
796 if (++trxSize > maxTransactionSize) {
797 if (!transaction.commit()) {
798 qCWarning(AKONADIDBMIGRATOR_LOG) <<
"Failed to commit transaction:" << destStore->
database().
lastError().
text();
805 emitTableProgress(table.name, ++processed, totalRows);
809 if (!transaction.commit()) {
810 qCWarning(AKONADIDBMIGRATOR_LOG) <<
"Failed to commit transaction:" << destStore->
database().
lastError().
text();
815 if (
const auto cnt = std::count_if(table.columns.begin(),
817 [](
const auto &col) {
818 return col.isAutoIncrement;
821 if (!syncAutoIncrementValue(sourceStore, destStore, table)) {
822 emitError(
i18nc(
"@info:status",
"Error: failed to update autoincrement value for table %1", table.name));
827 emitInfo(
i18nc(
"@info:status",
"Optimizing table %1...", table.name));
829 if (!analyzeTable(table.name, destStore)) {
830 emitError(
i18nc(
"@info:status",
"Error: failed to optimize table %1", table.name));
837void DbMigrator::emitInfo(
const QString &message)
847void DbMigrator::emitError(
const QString &message)
857void DbMigrator::emitProgress(
const QString &table,
int done,
int total)
861 [
this, table, done, total]() {
862 Q_EMIT progress(table, done, total);
867void DbMigrator::emitTableProgress(
const QString &table,
int done,
int total)
871 [
this, table, done, total]() {
872 Q_EMIT tableProgress(table, done, total);
877void DbMigrator::emitCompletion(
bool success)
882 Q_EMIT migrationCompleted(success);
887UIDelegate::Result DbMigrator::questionYesNo(
const QString &question)
889 UIDelegate::Result answer;
892 [
this, question, &answer]() {
894 answer = m_uiDelegate->questionYesNo(question);
901UIDelegate::Result DbMigrator::questionYesNoSkip(
const QString &question)
903 UIDelegate::Result answer;
906 [
this, question, &answer]() {
908 answer = m_uiDelegate->questionYesNoSkip(question);
914#include "moc_dbmigrator.cpp"
This class handles all the database access.
void close()
Closes the database connection.
QSqlDatabase database()
Returns the QSqlDatabase object.
A base class that provides an unique access layer to configuration and initialization of different da...
virtual void setup()
This method is called to setup initial database settings after a connection is established.
virtual void apply(QSqlDatabase &database)=0
This method applies the configured settings to the QtSql database instance.
virtual QString driverName() const =0
Returns the name of the used driver.
virtual QString databaseName() const =0
Returns the database name.
virtual void stopInternalServer()
This method is called to stop the external server.
virtual bool enableConstraintChecks(const QSqlDatabase &db)=0
Re-enables foreign key constraint checks.
virtual bool disableConstraintChecks(const QSqlDatabase &db)=0
Disables foreign key constraint checks.
virtual void setDatabasePath(const QString &path, QSettings &settings)=0
Set the path to the database file or directory.
virtual QString databasePath() const =0
Returns path to the database file or directory.
virtual bool startInternalServer()
This method is called to start an external server.
virtual bool useInternalServer() const =0
Returns whether an internal server needs to be used.
static DbIntrospector::Ptr createInstance(const QSqlDatabase &database)
Returns an introspector instance for a given database.
A helper class that describes a table for the DbInitializer.
QString i18nc(const char *context, const char *text, const TYPE &arg...)
Type type(const QSqlDatabase &db)
Returns the type of the given database object.
Helper integration between Akonadi and Qt.
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
QString path(const QString &relativePath)
KIOCORE_EXPORT void add(const QString &fileClass, const QString &directory)
QDBusConnectionInterface * interface() const const
QDBusConnection sessionBus()
QDBusReply< bool > isServiceRegistered(const QString &serviceName) const const
QDBusReply< QDBusConnectionInterface::RegisterServiceReply > registerService(const QString &serviceName, ServiceQueueOptions qoption, ServiceReplacementOptions roption)
QDBusReply< void > startService(const QString &name)
QDBusReply< bool > unregisterService(const QString &serviceName)
QString absolutePath() const const
bool isEmpty(Filters filters) const const
bool mkpath(const QString &dirPath) const const
Duration durationElapsed() const const
bool exists() const const
bool exists(const QString &path)
T value(qsizetype i) const const
QMetaObject::Connection connect(const QObject *sender, PointerToMemberFunction signal, Functor functor)
QSqlDatabase addDatabase(QSqlDriver *driver, const QString &connectionName)
QSqlError lastError() const const
QString text() const const
QString fromStdString(const std::string &str)
bool isEmpty() const const
QString toLower() const const
std::string toStdString() const const
QFuture< T > run(Function function,...)
QThread * create(Function &&f, Args &&... args)
void msleep(unsigned long msecs)