Skip to content
This repository was archived by the owner on Mar 4, 2023. It is now read-only.

Commit ebff4ab

Browse files
Skycoder42Skycoder42
committed
added migration code (untested)
1 parent 5b5b5de commit ebff4ab

File tree

6 files changed

+415
-4
lines changed

6 files changed

+415
-4
lines changed

‎doc/setup.dox‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ createPassive()) to create the instance.
1414
/*!
1515
@property QtDataSync::Setup::localDir
1616

17-
@default{`"./qtdatasync"`}
17+
@default{`"./qtdatasync/default"`}
1818

1919
The local directory is the heart of the datasync instance. It's where the actual data,
2020
configurations etc. are stored. A datasync instance is identified by it's local directory across

‎src/datasync/datasync.pro‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ HEADERS += \
3737
userexchangemanager_p.h \
3838
emitteradapter_p.h \
3939
changeemitter_p.h \
40-
signal_private_connect_p.h
40+
signal_private_connect_p.h \
41+
migrationhelper.h \
42+
migrationhelper_p.h
4143

4244
SOURCES += \
4345
localstore.cpp \
@@ -65,7 +67,8 @@ SOURCES += \
6567
accountmanager_p.cpp \
6668
userexchangemanager.cpp \
6769
emitteradapter.cpp \
68-
changeemitter.cpp
70+
changeemitter.cpp \
71+
migrationhelper.cpp
6972

7073
STATECHARTS += \
7174
connectorstatemachine.scxml

‎src/datasync/migrationhelper.cpp‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
#include "migrationhelper.h"
2+
#include "migrationhelper_p.h"
3+
#include "defaults.h"
4+
#include "localstore_p.h"
5+
#include "defaults_p.h"
6+
7+
#include <QtCore/QThreadPool>
8+
#include <QtCore/QStandardPaths>
9+
#include <QtCore/QDir>
10+
#include <QtCore/QLockFile>
11+
12+
#include <QtSql/QSqlDatabase>
13+
#include <QtSql/QSqlError>
14+
#include <QtSql/QSqlQuery>
15+
16+
using namespace QtDataSync;
17+
18+
const QString MigrationHelper::DefaultOldStorageDir = QStringLiteral("./qtdatasync_localstore");
19+
20+
MigrationHelper::MigrationHelper(QObject *parent) :
21+
MigrationHelper(DefaultSetup, parent)
22+
{}
23+
24+
MigrationHelper::MigrationHelper(const QString &setupName, QObject *parent) :
25+
QObject(parent),
26+
d(new MigrationHelperPrivate(setupName))
27+
{}
28+
29+
MigrationHelper::~MigrationHelper() {}
30+
31+
void MigrationHelper::startMigration(const QString &storageDir, MigrationFlags flags)
32+
{
33+
QThreadPool::globalInstance()->start(new MigrationRunnable {
34+
d->defaults,
35+
this,
36+
storageDir,
37+
flags
38+
});
39+
}
40+
41+
// ------------- PRIVATE IMPLEMENTATION -------------
42+
43+
MigrationHelperPrivate::MigrationHelperPrivate(const QString &setupName) :
44+
defaults(DefaultsPrivate::obtainDefaults(setupName))
45+
{}
46+
47+
48+
49+
MigrationRunnable::MigrationRunnable(const Defaults &defaults, MigrationHelper *helper, const QString &oldDir, MigrationHelper::MigrationFlags flags) :
50+
_defaults(defaults),
51+
_helper(helper),
52+
_oldDir(oldDir),
53+
_flags(flags),
54+
_logger(nullptr),
55+
_progress(0)
56+
{
57+
setAutoDelete(true);
58+
}
59+
60+
#define QTDATASYNC_LOG _logger
61+
62+
#define cleanDb() do { \
63+
db.close(); \
64+
db = QSqlDatabase(); \
65+
QSqlDatabase::removeDatabase(dbName); \
66+
} while(false)
67+
68+
#define dbError(errorSource) do { \
69+
logCritical().noquote() << "Database operation failed with error:" \
70+
<< errorSource.lastError().text(); \
71+
migrationDone(false); \
72+
cleanDb(); \
73+
return; \
74+
} while(false)
75+
76+
void MigrationRunnable::run()
77+
{
78+
//step 0: logging
79+
QObject scope;
80+
_logger = _defaults.createLogger("migration", &scope);
81+
82+
// check if the old storage exists
83+
QDir storageDir = QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation);
84+
if(!storageDir.cd(_oldDir)) {
85+
logInfo().noquote() << "The directory to be migrated does not exist. Assuming migration has already been done. Directory:"
86+
<< storageDir.absolutePath();
87+
migrationDone(true);
88+
return;
89+
}
90+
91+
//lock the setup
92+
QLockFile lockFile(storageDir.absoluteFilePath(QStringLiteral(".lock")));
93+
if(!lockFile.tryLock()) {
94+
qint64 pid;
95+
QString host, process;
96+
lockFile.getLockInfo(&pid, &host, &process);
97+
logCritical().noquote() << "The old storage directy is already locked by another process. Lock information:"
98+
<< "\n\tPID:" << pid
99+
<< "\n\tHost:" << host
100+
<< "\n\tApp-Name:" << process;
101+
migrationDone(false);
102+
return;
103+
}
104+
logDebug().noquote() << "Found and locked old storage directory:"
105+
<< storageDir.absolutePath();
106+
107+
//open the database
108+
auto dbName = QStringLiteral("database_migration_") + _defaults.setupName();
109+
auto db = QSqlDatabase::addDatabase(QStringLiteral("SQLITE"), dbName);
110+
db.setDatabaseName(storageDir.absoluteFilePath(QStringLiteral("store.db")));
111+
if(!db.open())
112+
dbError(db);
113+
114+
//count the number of things to migrate
115+
QSqlQuery countQuery(db);
116+
countQuery.prepare(QStringLiteral("SELECT Count(*) FROM DataIndex"));
117+
if(!countQuery.exec())
118+
dbError(countQuery);
119+
if(!countQuery.first())
120+
dbError(countQuery);
121+
auto migrationCount = countQuery.value(0).toInt();
122+
if(_flags.testFlag(MigrationHelper::MigrateRemoteConfig))
123+
migrationCount++;
124+
if(migrationCount == 0) {
125+
logInfo().noquote() << "Nothing found to be migrated";
126+
migrationDone(true);
127+
cleanDb();
128+
return;
129+
} else
130+
migrationPrepared(migrationCount);
131+
132+
//first step: migrate settings
133+
if(_flags.testFlag(MigrationHelper::MigrateRemoteConfig)) {
134+
QObject innerScope;
135+
auto currentSettings = _defaults.createSettings(&innerScope); //no group -> global
136+
auto oldSettings = new QSettings(storageDir.absoluteFilePath(QStringLiteral("config.ini")),
137+
QSettings::IniFormat,
138+
&innerScope);
139+
140+
//migrate key by key
141+
copyConf(oldSettings, QStringLiteral("RemoteConnector/remoteEnabled"),
142+
currentSettings, QStringLiteral("connector/enabled"));
143+
copyConf(oldSettings, QStringLiteral("RemoteConnector/remoteUrl"),
144+
currentSettings, QStringLiteral("connector/remote/url"));
145+
// copyConf(oldSettings, QStringLiteral("RemoteConnector/verifyPeer"),
146+
// currentSettings, QStringLiteral("connector/remote/verifyPeer")); //TODO add?
147+
copyConf(oldSettings, QStringLiteral("RemoteConnector/sharedSecret"),
148+
currentSettings, QStringLiteral("connector/remote/accessKey"));
149+
150+
copyConf(oldSettings, QStringLiteral("NetworkExchange/name"),
151+
currentSettings, QStringLiteral("connector/remote/accessKey"));
152+
153+
//special: copy headers
154+
oldSettings->beginGroup(QStringLiteral("RemoteConnector/headers"));
155+
currentSettings->beginGroup(QStringLiteral("connector/deviceName"));
156+
foreach(auto key, oldSettings->childKeys()) {
157+
copyConf(oldSettings, key,
158+
currentSettings, key);
159+
}
160+
currentSettings->endGroup();
161+
oldSettings->endGroup();
162+
163+
currentSettings->sync();
164+
migrationProgress();
165+
}
166+
167+
//next: migrate all the data step by step
168+
LocalStore store(_defaults, &scope);
169+
for(auto offset = 0; true; offset += 1000) {
170+
QSqlQuery entryQuery(db);
171+
entryQuery.prepare(QStringLiteral("SELECT DataIndex.Type, DataIndex.Key, File, Changed, SyncState.Type, SyncState.Key "
172+
"FROM DataIndex "
173+
"FULL OUTER JOIN SyncState "
174+
"ON DataIndex.Type = SyncState.Type AND DataIndex.Key = SyncState.Key "
175+
"ORDER BY DataIndex.Type, DataIndex.Key, SyncState.Type, SyncState.Key"
176+
"LIMIT 1000 OFFSET ?"));
177+
entryQuery.addBindValue(offset);
178+
if(!entryQuery.exec())
179+
dbError(entryQuery);
180+
181+
if(!entryQuery.first())
182+
break;
183+
do {
184+
//extract data
185+
auto type = entryQuery.value(0).toByteArray();
186+
QString key = entryQuery.value(1).toString();
187+
if(type.isEmpty()) {
188+
type = entryQuery.value(4).toByteArray();
189+
key = entryQuery.value(5).toString();
190+
}
191+
auto file = entryQuery.value(2).toString();
192+
auto state = entryQuery.value(3).toInt();
193+
194+
ObjectKey objKey(type, key);
195+
switch (state) {
196+
case 0: //unchanged
197+
if(!_flags.testFlag(MigrationHelper::MigrateChanged)) //when not migrating changes assume changed
198+
state = 1;
199+
Q_FALLTHROUGH();
200+
case 1: //changed
201+
{
202+
//find the storage directory
203+
auto tName = QString::fromUtf8("store/_" + QByteArray(type).toHex());
204+
auto mDir = storageDir;
205+
if(!mDir.cd(tName)) {
206+
logWarning() << "Failed to find file of stored data. Skipping"
207+
<< objKey;
208+
continue; //scope is the do-while loop
209+
}
210+
211+
//read the data file as json
212+
QFile inFile(mDir.absoluteFilePath(file));
213+
if(!inFile.open(QIODevice::ReadOnly)) {
214+
logWarning().noquote() << "Failed to open file of stored data. Skipping"
215+
<< objKey << "with file error:"
216+
<< inFile.errorString();
217+
continue; //scope is the do-while loop
218+
}
219+
auto data = QJsonDocument::fromBinaryData(inFile.readAll()).object();
220+
inFile.close();
221+
if(data.isEmpty()) {
222+
logWarning() << "Failed read file of stored data. Skipping"
223+
<< objKey;
224+
continue; //scope is the do-while loop
225+
}
226+
227+
//store in the store...
228+
auto scope = store.startSync(objKey);
229+
LocalStore::ChangeType type;
230+
quint64 version;
231+
QString fileName;
232+
tie(type, version, fileName, std::ignore) = store.loadChangeInfo(scope);
233+
if(type != LocalStore::NoExists && !_flags.testFlag(MigrationHelper::MigrateOverwriteData)) {
234+
logDebug() << "Skipping" << objKey << "as it would overwrite existing data";
235+
continue;
236+
}
237+
store.storeChanged(scope, version + 1, fileName, data, state == 1, type);
238+
store.commitSync(scope);
239+
logDebug() << "Migrated dataset" << objKey << "from the old store to the new one";
240+
break;
241+
}
242+
case 2: //deleted
243+
if(_flags.testFlag(MigrationHelper::MigrateChanged)) {
244+
//only when migrating changes, store the delete change
245+
auto scope = store.startSync(objKey);
246+
LocalStore::ChangeType type;
247+
quint64 version;
248+
tie(type, version, std::ignore, std::ignore) = store.loadChangeInfo(scope);
249+
if(type != LocalStore::NoExists && !_flags.testFlag(MigrationHelper::MigrateOverwriteData)) {
250+
logDebug() << "Skipping" << objKey << "as it would overwrite existing data";
251+
continue;
252+
}
253+
store.storeDeleted(scope, version, true, type);
254+
store.commitSync(scope);
255+
logDebug() << "Migrated deleted dataset" << objKey << "from the old store to the new one";
256+
break;
257+
}
258+
break;
259+
default:
260+
Q_UNREACHABLE();
261+
}
262+
263+
migrationProgress();
264+
} while(entryQuery.next());
265+
}
266+
267+
//cleanup merge
268+
cleanDb();
269+
lockFile.unlock();
270+
271+
//delete old stuff if wished
272+
if(_flags.testFlag(MigrationHelper::MigrateWithCleanup)) {
273+
if(!storageDir.removeRecursively())
274+
logWarning() << "Failed to remove storage directory. Unable to cleanup after migration";
275+
}
276+
277+
//complete migrate
278+
logInfo() << "Migration successfully completed";
279+
migrationDone(true);
280+
}
281+
282+
void MigrationRunnable::migrationPrepared(int count)
283+
{
284+
QMetaObject::invokeMethod(_helper, "migrationPrepared",
285+
Q_ARG(int, count));
286+
}
287+
288+
void MigrationRunnable::migrationProgress()
289+
{
290+
QMetaObject::invokeMethod(_helper, "migrationProgress",
291+
Q_ARG(int, ++_progress));
292+
}
293+
294+
void MigrationRunnable::migrationDone(bool ok)
295+
{
296+
QMetaObject::invokeMethod(_helper, "migrationDone",
297+
Q_ARG(bool, ok));
298+
}
299+
300+
void MigrationRunnable::copyConf(QSettings *old, const QString &oldKey, QSettings *current, const QString &newKey) const
301+
{
302+
if((_flags.testFlag(MigrationHelper::MigrateOverwriteConfig) || !current->contains(newKey)) &&
303+
old->contains(oldKey)) {
304+
current->setValue(newKey, old->value(oldKey));
305+
logDebug().noquote() << "Migrated old settings" << QString(old->group() + QLatin1Char('/') + oldKey)
306+
<< "to new settings as" << QString(current->group() + QLatin1Char('/') + newKey);
307+
}
308+
}

‎src/datasync/migrationhelper.h‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#ifndef QTDATASYNC_MIGRATIONHELPER_H
2+
#define QTDATASYNC_MIGRATIONHELPER_H
3+
4+
#include <QtCore/qobject.h>
5+
#include <QtCore/qscopedpointer.h>
6+
7+
#include "QtDataSync/qtdatasync_global.h"
8+
9+
namespace QtDataSync {
10+
11+
class MigrationHelperPrivate;
12+
class Q_DATASYNC_EXPORT MigrationHelper : public QObject
13+
{
14+
Q_OBJECT
15+
16+
public:
17+
enum MigrationFlag {
18+
MigrateData = 0x01,
19+
MigrateChanged = (0x02 | MigrateData),
20+
MigrateRemoteConfig = 0x04,
21+
MigrateAll = (MigrateData | MigrateChanged | MigrateRemoteConfig),
22+
23+
MigrateWithCleanup = 0x08,
24+
MigrateOverwriteConfig = 0x10,
25+
MigrateOverwriteData = 0x20
26+
};
27+
Q_DECLARE_FLAGS(MigrationFlags, MigrationFlag)
28+
Q_FLAG(MigrationFlags)
29+
30+
static const QString DefaultOldStorageDir;
31+
32+
explicit MigrationHelper(QObject *parent = nullptr);
33+
explicit MigrationHelper(const QString &setupName, QObject *parent = nullptr);
34+
~MigrationHelper();
35+
36+
public Q_SLOTS:
37+
void startMigration(const QString &storageDir = DefaultOldStorageDir, MigrationFlags flags = MigrationFlags(MigrateAll) | MigrateWithCleanup);
38+
39+
Q_SIGNALS:
40+
void migrationPrepared(int operations);
41+
void migrationProgress(int current);
42+
void migrationDone(bool ok);
43+
44+
private:
45+
QScopedPointer<MigrationHelperPrivate> d;
46+
};
47+
48+
}
49+
50+
Q_DECLARE_OPERATORS_FOR_FLAGS(QtDataSync::MigrationHelper::MigrationFlags)
51+
52+
#endif // QTDATASYNC_MIGRATIONHELPER_H

0 commit comments

Comments
 (0)