added basic migration test

Author: Skycoder42

src/datasync/localstore.cpp

@@ -718,7 +718,7 @@ void LocalStore::beginWriteTransaction(const ObjectKey &key, bool exclusive)
 
 void LocalStore::exec(QSqlQuery &query, const ObjectKey &key) const
 {
-	if(!query.exec()) {
+	if(!query.exec()) { //TODO do same for prepare
 			throw LocalStoreException(_defaults,
 									  key,
 									  query.executedQuery().simplified(),

src/datasync/migrationhelper.cpp

@@ -3,6 +3,7 @@
 #include "defaults.h"
 #include "localstore_p.h"
 #include "defaults_p.h"
+#include "remoteconnector_p.h"
 
 #include <QtCore/QThreadPool>
 #include <QtCore/QStandardPaths>
@@ -60,7 +61,8 @@ MigrationRunnable::MigrationRunnable(const Defaults &defaults, MigrationHelper *
 #define QTDATASYNC_LOG _logger
 
 #define cleanDb() do { \
-	db.close(); \
+	if(_flags.testFlag(MigrationHelper::MigrateData)) \
+		db.close(); \
 	db = QSqlDatabase(); \
 	QSqlDatabase::removeDatabase(dbName); \
 } while(false)
@@ -105,20 +107,45 @@ void MigrationRunnable::run()
 						 << storageDir.absolutePath();
 
 	//open the database
-	auto dbName = QStringLiteral("database_migration_") + _defaults.setupName();
-	auto db = QSqlDatabase::addDatabase(QStringLiteral("SQLITE"), dbName);
-	db.setDatabaseName(storageDir.absoluteFilePath(QStringLiteral("store.db")));
-	if(!db.open())
-		dbError(db);
+	QString dbName = QStringLiteral("database_migration_") + _defaults.setupName();
+	auto db = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), dbName);
+	if(_flags.testFlag(MigrationHelper::MigrateData)) {
+		db.setDatabaseName(storageDir.absoluteFilePath(QStringLiteral("store.db")));
+		if(!db.open())
+			dbError(db);
+		else {
+			if(!db.tables().contains(QStringLiteral("DataIndex")) ||
+			   !db.tables().contains(QStringLiteral("SyncState"))) {
+				logWarning() << "Missing tables in database. Either DataIndex or SyncState could not be found";
+				cleanDb();
+				migrationDone(false);
+				return;
+			} else
+				logDebug() << "Successfully opened database";
+		}
+	}
 
 	//count the number of things to migrate
-	QSqlQuery countQuery(db);
-	countQuery.prepare(QStringLiteral("SELECT Count(*) FROM DataIndex"));
-	if(!countQuery.exec())
-		dbError(countQuery);
-	if(!countQuery.first())
-		dbError(countQuery);
-	auto migrationCount = countQuery.value(0).toInt();
+	int migrationCount = 0;
+	if(_flags.testFlag(MigrationHelper::MigrateData)) {
+		QSqlQuery countQuery(db);
+		if(!countQuery.exec(QStringLiteral("SELECT Count(*) FROM ("
+										   "	SELECT i.Type AS Type, i.Key AS Key "
+										   "	FROM DataIndex i "
+										   "	LEFT JOIN SyncState s "
+										   "	ON i.Type = s.Type AND i.Key = s.Key "
+										   "	UNION "
+										   "	SELECT s.Type AS Type, s.Key AS Key "
+										   "	FROM SyncState s "
+										   "	LEFT JOIN DataIndex i "
+										   "	ON s.Type = i.Type AND s.Key = i.Key "
+										   "	WHERE i.Type IS NULL "
+										   ")")))
+			dbError(countQuery);
+		if(!countQuery.first())
+			dbError(countQuery);
+		migrationCount = countQuery.value(0).toInt();
+	}
 	if(_flags.testFlag(MigrationHelper::MigrateRemoteConfig))
 		migrationCount++;
 	if(migrationCount == 0) {
@@ -139,20 +166,20 @@ void MigrationRunnable::run()
 
 		//migrate key by key
 		copyConf(oldSettings, QStringLiteral("RemoteConnector/remoteEnabled"),
-				 currentSettings, QStringLiteral("connector/enabled"));
+				 currentSettings, QStringLiteral("connector/") + RemoteConnector::keyRemoteEnabled);
 		copyConf(oldSettings, QStringLiteral("RemoteConnector/remoteUrl"),
-				 currentSettings, QStringLiteral("connector/remote/url"));
+				 currentSettings, QStringLiteral("connector/") + RemoteConnector::keyRemoteUrl);
 //		copyConf(oldSettings, QStringLiteral("RemoteConnector/verifyPeer"),
 //				 currentSettings, QStringLiteral("connector/remote/verifyPeer")); //TODO add?
 		copyConf(oldSettings, QStringLiteral("RemoteConnector/sharedSecret"),
-				 currentSettings, QStringLiteral("connector/remote/accessKey"));
+				 currentSettings, QStringLiteral("connector/") + RemoteConnector::keyRemoteAccessKey);
 
 		copyConf(oldSettings, QStringLiteral("NetworkExchange/name"),
-				 currentSettings, QStringLiteral("connector/remote/accessKey"));
+				 currentSettings, QStringLiteral("connector/") + RemoteConnector::keyDeviceName);
 
 		//special: copy headers
 		oldSettings->beginGroup(QStringLiteral("RemoteConnector/headers"));
-		currentSettings->beginGroup(QStringLiteral("connector/deviceName"));
+		currentSettings->beginGroup(QStringLiteral("connector/") + RemoteConnector::keyRemoteHeaders);
 		foreach(auto key, oldSettings->childKeys()) {
 			copyConf(oldSettings, key,
 					 currentSettings, key);
@@ -165,103 +192,120 @@ void MigrationRunnable::run()
 	}
 
 	//next: migrate all the data step by step
-	LocalStore store(_defaults, &scope);
-	for(auto offset = 0; true; offset += 1000) {
-		QSqlQuery entryQuery(db);
-		entryQuery.prepare(QStringLiteral("SELECT DataIndex.Type, DataIndex.Key, File, Changed, SyncState.Type, SyncState.Key "
-										  "FROM DataIndex "
-										  "FULL OUTER JOIN SyncState "
-										  "ON DataIndex.Type = SyncState.Type AND DataIndex.Key = SyncState.Key "
-										  "ORDER BY DataIndex.Type, DataIndex.Key, SyncState.Type, SyncState.Key"
-										  "LIMIT 1000 OFFSET ?"));
-		entryQuery.addBindValue(offset);
-		if(!entryQuery.exec())
-			dbError(entryQuery);
-
-		if(!entryQuery.first())
-			break;
-		do {
-			//extract data
-			auto type = entryQuery.value(0).toByteArray();
-			QString key = entryQuery.value(1).toString();
-			if(type.isEmpty()) {
-				type = entryQuery.value(4).toByteArray();
-				key = entryQuery.value(5).toString();
-			}
-			auto file = entryQuery.value(2).toString();
-			auto state = entryQuery.value(3).toInt();
-
-			ObjectKey objKey(type, key);
-			switch (state) {
-			case 0: //unchanged
-				if(!_flags.testFlag(MigrationHelper::MigrateChanged)) //when not migrating changes assume changed
-					state = 1;
-				Q_FALLTHROUGH();
-			case 1: //changed
-			{
-				//find the storage directory
-				auto tName = QString::fromUtf8("store/_" + QByteArray(type).toHex());
-				auto mDir = storageDir;
-				if(!mDir.cd(tName)) {
-					logWarning() << "Failed to find file of stored data. Skipping"
-								 << objKey;
-					continue; //scope is the do-while loop
-				}
+	if(_flags.testFlag(MigrationHelper::MigrateData)) {
+		LocalStore store(_defaults, &scope);
+		for(auto offset = 0; true; offset += 100) {
+			QSqlQuery entryQuery(db);
+			if(!entryQuery.prepare(QStringLiteral("SELECT Type, Key, File, Changed FROM ( "
+												  "		SELECT i.Type AS Type, i.Key AS Key, i.File AS File, s.Changed AS Changed "
+												  "		FROM DataIndex i "
+												  "		LEFT JOIN SyncState s "
+												  "		ON i.Type = s.Type AND i.Key = s.Key "
+												  "		UNION "
+												  "		SELECT s.Type AS Type, s.Key AS Key, i.File AS File, s.Changed AS Changed "
+												  "		FROM SyncState s "
+												  "		LEFT JOIN DataIndex i "
+												  "		ON s.Type = i.Type AND s.Key = i.Key "
+												  "		WHERE i.Type IS NULL "
+												  ") ORDER BY Type, Key "
+												  "LIMIT 100 OFFSET ?")))
+				dbError(entryQuery);
+			entryQuery.addBindValue(offset);
+			if(!entryQuery.exec())
+				dbError(entryQuery);
+
+			if(!entryQuery.first())
+				break;
+			do {
+				//extract data
+				ObjectKey key {
+					entryQuery.value(0).toByteArray(),
+					entryQuery.value(1).toString()
+				};
+				QString file = entryQuery.value(2).toString() + QStringLiteral(".dat");
+				auto state = entryQuery.value(3).toInt();
+
+				switch (state) {
+				case 0: //unchanged
+					if(!_flags.testFlag(MigrationHelper::MigrateChanged)) //when not migrating changes assume changed
+						state = 1;
+					Q_FALLTHROUGH();
+				case 1: //changed
+				{
+					//find the storage directory
+					auto tName = QString::fromUtf8("store/_" + key.typeName.toHex());
+					auto mDir = storageDir;
+					if(!mDir.cd(tName)) {
+						logWarning() << "Failed to find file of stored data. Skipping"
+									 << key << " with missing path:" << tName;
+						migrationProgress();
+						continue; //scope is the do-while loop
+					}
 
-				//read the data file as json
-				QFile inFile(mDir.absoluteFilePath(file));
-				if(!inFile.open(QIODevice::ReadOnly)) {
-					logWarning().noquote() << "Failed to open file of stored data. Skipping"
-										   << objKey << "with file error:"
-										   << inFile.errorString();
-					continue; //scope is the do-while loop
-				}
-				auto data = QJsonDocument::fromBinaryData(inFile.readAll()).object();
-				inFile.close();
-				if(data.isEmpty()) {
-					logWarning() << "Failed read file of stored data. Skipping"
-								 << objKey;
-					continue; //scope is the do-while loop
-				}
+					//read the data file as json
+					QFile inFile(mDir.absoluteFilePath(file));
+					if(!inFile.exists()) {
+						logWarning() << "Failed to find file of stored data. Skipping"
+									 << key << " with missing path:" << inFile.fileName();
+						migrationProgress();
+						continue; //scope is the do-while loop
+					}
+					if(!inFile.open(QIODevice::ReadOnly)) {
+						logWarning().noquote() << "Failed to open file of stored data. Skipping"
+											   << key << "with file error:"
+											   << inFile.errorString();
+						migrationProgress();
+						continue; //scope is the do-while loop
+					}
+					auto data = QJsonDocument::fromBinaryData(inFile.readAll()).object();
+					inFile.close();
+					if(data.isEmpty()) {
+						logWarning() << "Failed read file of stored data. Skipping"
+									 << key;
+						migrationProgress();
+						continue; //scope is the do-while loop
+					}
 
-				//store in the store...
-				auto scope = store.startSync(objKey);
-				LocalStore::ChangeType type;
-				quint64 version;
-				QString fileName;
-				tie(type, version, fileName, std::ignore) = store.loadChangeInfo(scope);
-				if(type != LocalStore::NoExists && !_flags.testFlag(MigrationHelper::MigrateOverwriteData)) {
-					logDebug() << "Skipping" << objKey << "as it would overwrite existing data";
-					continue;
-				}
-				store.storeChanged(scope, version + 1, fileName, data, state == 1, type);
-				store.commitSync(scope);
-				logDebug() << "Migrated dataset" << objKey << "from the old store to the new one";
-				break;
-			}
-			case 2: //deleted
-				if(_flags.testFlag(MigrationHelper::MigrateChanged)) {
-					//only when migrating changes, store the delete change
-					auto scope = store.startSync(objKey);
+					//store in the store...
+					auto scope = store.startSync(key);
 					LocalStore::ChangeType type;
 					quint64 version;
-					tie(type, version, std::ignore, std::ignore) = store.loadChangeInfo(scope);
+					QString fileName;
+					tie(type, version, fileName, std::ignore) = store.loadChangeInfo(scope);
 					if(type != LocalStore::NoExists && !_flags.testFlag(MigrationHelper::MigrateOverwriteData)) {
-						logDebug() << "Skipping" << objKey << "as it would overwrite existing data";
+						logDebug() << "Skipping" << key << "as it would overwrite existing data";
+						migrationProgress();
 						continue;
 					}
-					store.storeDeleted(scope, version, true, type);
+					store.storeChanged(scope, version + 1, fileName, data, state == 1, type);
 					store.commitSync(scope);
-					logDebug() << "Migrated deleted dataset" << objKey << "from the old store to the new one";
+					logDebug() << "Migrated dataset" << key << "from the old store to the new one";
+					migrationProgress();
 					break;
 				}
-				break;
-			default:
-				Q_UNREACHABLE();
-			}
-
-			migrationProgress();
-		} while(entryQuery.next());
+				case 2: //deleted
+					if(_flags.testFlag(MigrationHelper::MigrateChanged)) {
+						//only when migrating changes, store the delete change
+						auto scope = store.startSync(key);
+						LocalStore::ChangeType type;
+						quint64 version;
+						tie(type, version, std::ignore, std::ignore) = store.loadChangeInfo(scope);
+						if(type != LocalStore::NoExists && !_flags.testFlag(MigrationHelper::MigrateOverwriteData)) {
+							logDebug() << "Skipping" << key << "as it would overwrite existing data";
+							migrationProgress();
+							continue;
+						}
+						store.storeDeleted(scope, version + 1, true, type);
+						store.commitSync(scope);
+						logDebug() << "Migrated deleted dataset" << key << "from the old store to the new one";
+					}
+					migrationProgress();
+					break;
+				default:
+					Q_UNREACHABLE();
+				}
+			} while(entryQuery.next());
+		}
 	}
 
 	//cleanup merge
@@ -304,5 +348,8 @@ void MigrationRunnable::copyConf(QSettings *old, const QString &oldKey, QSetting
 		current->setValue(newKey, old->value(oldKey));
 		logDebug().noquote() << "Migrated old settings" << QString(old->group() + QLatin1Char('/') + oldKey)
 							 << "to new settings as" << QString(current->group() + QLatin1Char('/') + newKey);
+	} else {
+		logDebug().noquote() << "Skipping migration of key" << QString(old->group() + QLatin1Char('/') + oldKey)
+							 << "to new settings as" << QString(current->group() + QLatin1Char('/') + newKey);
 	}
 }

src/datasync/migrationhelper_p.h

@@ -2,6 +2,7 @@
 #define QTDATASYNC_MIGRATIONHELPER_P_H
 
 #include <QtCore/QRunnable>
+#include <QtCore/QPointer>
 
 #include "qtdatasync_global.h"
 #include "migrationhelper.h"