From 7225ebe2ce42e5b3fb0a350ea27b60c3fe43acea Mon Sep 17 00:00:00 2001 From: Filip Bucek Date: Thu, 28 Nov 2019 14:20:11 +0100 Subject: [PATCH 01/13] suggestion how to proceed with next steps ( prepare data ) Not validated i cannot check anything remotely --- src/colnod/colnodapi.cpp | 7 +++++++ src/colnod/data.h | 6 ++++++ src/colnod/syncmanager.cpp | 20 +++++++++++++++++--- src/colnod/syncmanager.h | 13 +++++++++++++ test/auto/syncmanager/tst_syncmanager.cpp | 10 +++++++++- 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/colnod/colnodapi.cpp b/src/colnod/colnodapi.cpp index 4340c8b..906fb0b 100644 --- a/src/colnod/colnodapi.cpp +++ b/src/colnod/colnodapi.cpp @@ -14,6 +14,13 @@ ColnodAPI::ColnodAPI(serversdk::RequestManager* reqMan, const qint64 &lastSync, // ----------------------------------- } +/** + * @brief Downloads new data from Colnod -> emits downloadDataFinished signal + * @param time + * + * @return signal with nullptr when something went wrong + * @return signal with data when valid data returned + */ void ColnodAPI::downloadData(const Timestamp &time) { // Have to get Login diff --git a/src/colnod/data.h b/src/colnod/data.h index 340c9eb..41a2491 100644 --- a/src/colnod/data.h +++ b/src/colnod/data.h @@ -37,3 +37,9 @@ public: QList deletedSubjects; }; +class PreparedData +{ + // TODO: make unique_ptr ( neeed because of ColnodAPI ) + std::shared_ptr databaseData; + std::shared_ptr colnodData; +}; diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 9cf9cb3..5244796 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -66,16 +66,25 @@ bool SyncManager::startSynchronization() connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr data) { if (!data) { qCritical(syncmanager) << "Not possible to get correct data from colnod"; + QCoreApplication::exit(-1); return; } - - - // NOW I HAVE DATA FROM COLNOD + // NOW I SHOULD HAVE VALID DATA FROM COLNOD // I can process data qDebug(syncmanager) << "I have colnod and database data"; qDebug(syncmanager) << "updated token count" << data->updatedTokens.count(); + std::unique_ptr preparedData = prepareDataNew(databaseData, colnodData); + if (!preparedData) { + qCritical(syncmanager) << "Not possible to get correct data from colnod"; + QCoreApplication::exit(-1); + return; + } + + colnod->processData(preparedData->colnodData); + + }); return true; @@ -121,6 +130,11 @@ bool SyncManager::startSynchronization() return true; } +std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData) +{ + return nullptr; +} + bool SyncManager::downloadData() { bool res = true; diff --git a/src/colnod/syncmanager.h b/src/colnod/syncmanager.h index 07c92e0..5fbfa31 100644 --- a/src/colnod/syncmanager.h +++ b/src/colnod/syncmanager.h @@ -8,12 +8,25 @@ Q_DECLARE_LOGGING_CATEGORY(syncmanager) class SyncManager : public QObject { Q_OBJECT + friend class SyncManagerTest; public: explicit SyncManager(bool dryMode); bool databaseInitialization(); bool startSynchronization(); + // New approach +private: + /** + * @brief Preapre data to be pushed to database and colnod from actual downloaded data + * @param data from database + * @param data from colnod + * @return prepared data to be pushed into colnod and db + * + * @todo rename when removed old aproach + */ + std::unique_ptr prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData); + private: bool downloadData(); bool prepareData(); diff --git a/test/auto/syncmanager/tst_syncmanager.cpp b/test/auto/syncmanager/tst_syncmanager.cpp index ad97eff..ff3fbaf 100644 --- a/test/auto/syncmanager/tst_syncmanager.cpp +++ b/test/auto/syncmanager/tst_syncmanager.cpp @@ -8,16 +8,24 @@ class SyncManagerTest : public QObject Q_OBJECT private slots: void sync_init_test(); + void prepareData_test(); }; void SyncManagerTest::sync_init_test() { SyncManager sync(true); - QVERIFY(sync.databaseInitialization()); } +void SyncManagerTest::prepareData_test() +{ + SyncManager sync(true); + auto dbData = std::make_shared(); + auto colnodData = std::make_shared(); + QVERIFY2(sync.prepareDataNew(dbData, colnodData), "finish unitest"); +} + QTEST_MAIN(SyncManagerTest) #include "tst_syncmanager.moc" From 3fbeb3130cb0909f99292d7c2f2308cdcfe16889 Mon Sep 17 00:00:00 2001 From: Filip Bucek Date: Thu, 28 Nov 2019 14:31:24 +0100 Subject: [PATCH 02/13] updated sever-sdk + fixed unused warnings + added explicit to TimeStamp constructor --- server-sdk | 2 +- src/colnod/colnodapi.h | 2 +- src/colnod/data.h | 2 +- src/colnod/syncmanager.cpp | 7 ++++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/server-sdk b/server-sdk index 2041fed..a818055 160000 --- a/server-sdk +++ b/server-sdk @@ -1 +1 @@ -Subproject commit 2041fed9fab14658ceb574edaf5e4d31ed81e23a +Subproject commit a818055b75dc0de97e26be44cbbfd2e94516c09e diff --git a/src/colnod/colnodapi.h b/src/colnod/colnodapi.h index af79297..a8e799c 100644 --- a/src/colnod/colnodapi.h +++ b/src/colnod/colnodapi.h @@ -24,7 +24,7 @@ class Timestamp : public serversdk::DatabaseTable { TABLE_PROPERTY(qint64, timestamp, set_timestamp, 0) public: Timestamp() = default; - Timestamp(const Timestamp &time) + explicit Timestamp(const Timestamp &time) { this->set_timestamp(time.timestamp()); } diff --git a/src/colnod/data.h b/src/colnod/data.h index 41a2491..5ef4827 100644 --- a/src/colnod/data.h +++ b/src/colnod/data.h @@ -37,7 +37,7 @@ public: QList deletedSubjects; }; -class PreparedData +struct PreparedData { // TODO: make unique_ptr ( neeed because of ColnodAPI ) std::shared_ptr databaseData; diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 5244796..cccecec 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -82,15 +82,14 @@ bool SyncManager::startSynchronization() return; } - colnod->processData(preparedData->colnodData); - - + // NOT FINISHED scolnod->processData(preparedData->colnodData); }); return true; //downloadData(); auto colnodDataFinished = [=](std::shared_ptr data) { + Q_UNUSED(data) qDebug() << "Now I can check that I have all data I need"; }; @@ -132,6 +131,8 @@ bool SyncManager::startSynchronization() std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData) { + Q_UNUSED(databaseData); + Q_UNUSED(colnodData); return nullptr; } From 0dea65a0b65c9a363306caa3bc371392b571119d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Thu, 28 Nov 2019 15:02:45 +0100 Subject: [PATCH 03/13] WIP: prepare data --- src/colnod/syncmanager.cpp | 42 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index cccecec..7278037 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -131,8 +131,46 @@ bool SyncManager::startSynchronization() std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData) { - Q_UNUSED(databaseData); - Q_UNUSED(colnodData); + auto preperedColnodData = std::make_shared(); + auto preperedDatabaseData = std::make_shared(); + + QList conflicts; + QList alreadySynced; + + for (const auto &colnodToken : colnodData->updatedTokens) { + + // not conflict so add to prepered tokens + if (!databaseData->updatedTokens.contains(colnodToken->id())) { + preperedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken); + continue; + } + + // change in colnod is newer + if (databaseData->updatedTokens.value(colnodToken->id()) < colnodToken) { + preperedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken); + } + + // updated in db later + if (databaseData->updatedTokens.value(colnodToken->id()) > colnodToken) { + conflicts.append(colnodToken->id()); + } + } + auto token = database->getToken(colnodToken->id()); + if ( token ) + // alredy synced ( tokens are same ) + if (token == colnodToken) + alreadySynced.append(colnodToken->id()); + } + // remove tokens which should be not updated + for (const auto &id : conflicts) + colnodData->updatedTokens.remove(id); + for (const auto &id : alreadySynced) + colnodData->updatedTokens.remove(id); + + qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:"; + qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE TOO"; + qInfo(syncmanager) << tab << tab << alreadySynced.size() << "TOKENS WERE ALREADY SYNCED"; + return nullptr; } From 53015d839aa99536a8d5be7a3a8c301d9cc10430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Thu, 28 Nov 2019 15:16:34 +0100 Subject: [PATCH 04/13] WIP: prepare data --- src/colnod/syncmanager.cpp | 46 +++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 7278037..4d7ed4d 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -149,27 +149,53 @@ std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr< if (databaseData->updatedTokens.value(colnodToken->id()) < colnodToken) { preperedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken); } - - // updated in db later - if (databaseData->updatedTokens.value(colnodToken->id()) > colnodToken) { - conflicts.append(colnodToken->id()); - } + else { + conflicts.append(colnodToken->id()); } + auto token = database->getToken(colnodToken->id()); if ( token ) // alredy synced ( tokens are same ) if (token == colnodToken) alreadySynced.append(colnodToken->id()); } - // remove tokens which should be not updated - for (const auto &id : conflicts) - colnodData->updatedTokens.remove(id); - for (const auto &id : alreadySynced) - colnodData->updatedTokens.remove(id); + + for (const auto &id : colnodData->deletedTokens) { + if (!database->isAlreadyDeleted(id)) { + preperedColnodData->deletedTokens.append(id); + } + } qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:"; qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE TOO"; qInfo(syncmanager) << tab << tab << alreadySynced.size() << "TOKENS WERE ALREADY SYNCED"; + qInfo(syncmanager) << tab << tab << colnodData->updatedTokens.size() - preperedColnodData->deletedTokens.size() << "TOKENS WERE ALREADY DELETED"; + qInfo(syncmanager) << dash; + + for (const auto &databaseToken : databaseData->updatedTokens) { + if (!colnodData->updatedTokens.contains(databaseToken->id())) { + preperedDatabaseData->updatedTokens.insert(databaseToken->id(), databaseToken); + } + if (databaseToken < colnodData->updatedTokens.value(databaseToken->id())) + conflicts.append(databaseToken->id()); + } + else { + // restriction cauesed by colnod rest api + if (databaseToken->description() != "mobile") { + qCritical(syncmanager) << "Token:" << databaseToken->id() << "doesn't have 'mobile' in description, can't be "; + conflicts.append(databaseToken->id()); + } + } + } + for (const auto &id : conflicts) + colnodData->updatedTokens.remove(id); + + qInfo(syncmanager) << tab << "-> DATABASE DATA - CHANGES:"; + qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN IN COLNOD TOO"; + qInfo(syncmanager) << dash; + + qInfo(syncmanager) << "PREPARING DATA -> FINISHED"; + qInfo(syncmanager) << equals; return nullptr; } From 1945e7e002077a7d756412ab49343e196048e028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Thu, 28 Nov 2019 15:49:49 +0100 Subject: [PATCH 05/13] finished prepere data, but not hadling new tokens --- src/colnod/data.h | 8 +++++ src/colnod/syncmanager.cpp | 61 +++++++++++++++++++------------------- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/colnod/data.h b/src/colnod/data.h index 5ef4827..cd6d512 100644 --- a/src/colnod/data.h +++ b/src/colnod/data.h @@ -39,6 +39,14 @@ public: struct PreparedData { +public: + PreparedData() = default; + PreparedData(std::shared_ptr database, std::shared_ptr colnod) + :databaseData(database), colnodData(colnod) + { + } + + // TODO: make unique_ptr ( neeed because of ColnodAPI ) std::shared_ptr databaseData; std::shared_ptr colnodData; diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 4d7ed4d..1529509 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -49,7 +49,7 @@ bool SyncManager::startSynchronization() qInfo(syncmanager) << equals; qInfo(syncmanager) << "SYNCHRONIZATION BEGIN"; - // TODO: check error + // TODO: solve this in prepare data database->initNewTokens(); @@ -63,8 +63,8 @@ bool SyncManager::startSynchronization() colnod->downloadData(Timestamp(syncBegin)); - connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr data) { - if (!data) { + connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr colnodData) { + if (!colnodData) { qCritical(syncmanager) << "Not possible to get correct data from colnod"; QCoreApplication::exit(-1); return; @@ -73,7 +73,7 @@ bool SyncManager::startSynchronization() // NOW I SHOULD HAVE VALID DATA FROM COLNOD // I can process data qDebug(syncmanager) << "I have colnod and database data"; - qDebug(syncmanager) << "updated token count" << data->updatedTokens.count(); + qDebug(syncmanager) << "updated token count" << colnodData->updatedTokens.count(); std::unique_ptr preparedData = prepareDataNew(databaseData, colnodData); if (!preparedData) { @@ -88,11 +88,6 @@ bool SyncManager::startSynchronization() return true; //downloadData(); - auto colnodDataFinished = [=](std::shared_ptr data) { - Q_UNUSED(data) - qDebug() << "Now I can check that I have all data I need"; - }; - //connect(this, &SyncManager::dataReady, processDownloadedData); @@ -131,8 +126,8 @@ bool SyncManager::startSynchronization() std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData) { - auto preperedColnodData = std::make_shared(); - auto preperedDatabaseData = std::make_shared(); + auto preparedColnodData = std::make_shared(); + auto preparedDatabaseData = std::make_shared(); QList conflicts; QList alreadySynced; @@ -141,13 +136,13 @@ std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr< // not conflict so add to prepered tokens if (!databaseData->updatedTokens.contains(colnodToken->id())) { - preperedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken); + preparedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken); continue; } // change in colnod is newer if (databaseData->updatedTokens.value(colnodToken->id()) < colnodToken) { - preperedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken); + preparedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken); } else { conflicts.append(colnodToken->id()); @@ -162,42 +157,46 @@ std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr< for (const auto &id : colnodData->deletedTokens) { if (!database->isAlreadyDeleted(id)) { - preperedColnodData->deletedTokens.append(id); + preparedColnodData->deletedTokens.append(id); } } qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:"; - qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE TOO"; + qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE AFTER"; qInfo(syncmanager) << tab << tab << alreadySynced.size() << "TOKENS WERE ALREADY SYNCED"; - qInfo(syncmanager) << tab << tab << colnodData->updatedTokens.size() - preperedColnodData->deletedTokens.size() << "TOKENS WERE ALREADY DELETED"; + qInfo(syncmanager) << tab << tab << colnodData->deletedTokens.size() - preparedColnodData->deletedTokens.size() << "TOKENS WERE ALREADY DELETED"; qInfo(syncmanager) << dash; for (const auto &databaseToken : databaseData->updatedTokens) { + + // restriction cauesed by colnod rest api + if (databaseToken->description() != "mobile") { + qCritical(syncmanager) << "Token:" << databaseToken->id() << "doesn't have 'mobile' in description, can't be "; + continue; + } + + // no conflict if (!colnodData->updatedTokens.contains(databaseToken->id())) { - preperedDatabaseData->updatedTokens.insert(databaseToken->id(), databaseToken); - } - if (databaseToken < colnodData->updatedTokens.value(databaseToken->id())) - conflicts.append(databaseToken->id()); - } - else { - // restriction cauesed by colnod rest api - if (databaseToken->description() != "mobile") { - qCritical(syncmanager) << "Token:" << databaseToken->id() << "doesn't have 'mobile' in description, can't be "; - conflicts.append(databaseToken->id()); - } + preparedDatabaseData->updatedTokens.insert(databaseToken->id(), databaseToken); + continue; } + + // database change is newer + if (databaseToken > colnodData->updatedTokens.value(databaseToken->id())) { + preparedDatabaseData->updatedTokens.insert(databaseToken->id(), databaseToken); + } } - for (const auto &id : conflicts) - colnodData->updatedTokens.remove(id); qInfo(syncmanager) << tab << "-> DATABASE DATA - CHANGES:"; - qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN IN COLNOD TOO"; + qInfo(syncmanager) << tab << tab << databaseData->updatedTokens.size() - preparedColnodData->updatedTokens.size() << "TOKENS WERE ADDED/UPDATED IN COLNOD AFTER"; qInfo(syncmanager) << dash; qInfo(syncmanager) << "PREPARING DATA -> FINISHED"; qInfo(syncmanager) << equals; - return nullptr; + + auto preparedData = std::make_unique(preparedDatabaseData, preparedColnodData); + return preparedData; } bool SyncManager::downloadData() From 53697249c634d3f6ffd1649d7222d0c3c5c1abf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Thu, 28 Nov 2019 16:00:22 +0100 Subject: [PATCH 06/13] Add handle of init new tokens --- src/colnod/syncmanager.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 1529509..9e630c2 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -99,7 +99,7 @@ bool SyncManager::startSynchronization() qInfo(syncmanager) << "PROCESSING DATA -> FINISHED"; qInfo(syncmanager) << equals; - updateLastUpdate(syncBegin); + updateLastUpdate(syncBegin); // TODO: change to update all attrinutes connect(this, &SyncManager::syncFinished, [=](){ if (database->isValid() && colnod->isValid()) { database->setLastSync(syncBegin); @@ -168,6 +168,9 @@ std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr< qInfo(syncmanager) << dash; for (const auto &databaseToken : databaseData->updatedTokens) { + if (databaseToken->name().isEmpty()) { + initNewToken(databaseToken); + } // restriction cauesed by colnod rest api if (databaseToken->description() != "mobile") { @@ -199,6 +202,16 @@ std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr< return preparedData; } +void SyncManager::initNewToken(std::shared_ptr token) +{ + qint16 num = database->getNumOfTokensOfSubject(token->subjectId()); + token->setName(database->getSubjectNameByTokenId(token->id()) + "_mobile_" + QString::number(num + 1)); + token->setDescription("mobile"); + token->setTokenData(token->tokenData().toUpper()); + + qInfo(syncmanager).noquote() << Message::massageTemplate("Inicializing new", "token", "sync manager", token->name()); +} + bool SyncManager::downloadData() { bool res = true; From dd0f8fe26736172a4d28e385c9680467fb700674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Thu, 28 Nov 2019 16:02:35 +0100 Subject: [PATCH 07/13] fix cpp check, and unsaved stuff from previous commit --- src/colnod/data.h | 2 +- src/colnod/databaseapi.h | 5 +++-- src/colnod/syncmanager.h | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/colnod/data.h b/src/colnod/data.h index cd6d512..a0f607a 100644 --- a/src/colnod/data.h +++ b/src/colnod/data.h @@ -41,7 +41,7 @@ struct PreparedData { public: PreparedData() = default; - PreparedData(std::shared_ptr database, std::shared_ptr colnod) + PreparedData(const std::shared_ptr &database, const std::shared_ptr &colnod) :databaseData(database), colnodData(colnod) { } diff --git a/src/colnod/databaseapi.h b/src/colnod/databaseapi.h index 34b125d..8668b2f 100644 --- a/src/colnod/databaseapi.h +++ b/src/colnod/databaseapi.h @@ -48,13 +48,14 @@ public: bool isValid(); + qint16 getNumOfTokensOfSubject(const QString &id); + QString getSubjectNameByTokenId(const QString &tokenId); + private: int exists(const QString &id, const QString &tableName); QSqlQuery getUpdatedTokens(); std::shared_ptr composeData(const QSqlQuery &updatedQuery, QSqlQuery &deletedQuery); - qint16 getNumOfTokensOfSubject(const QString &id); - QString getSubjectNameByTokenId(const QString &tokenId); signals: void downloadDataFinished(std::shared_ptr = std::shared_ptr()); diff --git a/src/colnod/syncmanager.h b/src/colnod/syncmanager.h index 5fbfa31..0d2719b 100644 --- a/src/colnod/syncmanager.h +++ b/src/colnod/syncmanager.h @@ -28,6 +28,8 @@ private: std::unique_ptr prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData); private: + void initNewToken(std::shared_ptr token); + bool downloadData(); bool prepareData(); bool processData(); From 649b494682a96849057bc9553b20bef9774264e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Mon, 2 Dec 2019 10:20:16 +0100 Subject: [PATCH 08/13] prepering next step: process data --- src/colnod/colnodapi.cpp | 2 +- src/colnod/colnodapi.h | 2 +- src/colnod/syncmanager.cpp | 11 ++++++----- src/colnod/syncmanager.h | 7 ------- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/colnod/colnodapi.cpp b/src/colnod/colnodapi.cpp index 1433581..279b38f 100644 --- a/src/colnod/colnodapi.cpp +++ b/src/colnod/colnodapi.cpp @@ -39,7 +39,7 @@ void ColnodAPI::downloadData(const Timestamp &time) if (!checkRequest(request)) { qWarning(colnod) << "ERROR during calling" << request->url() << ", request unsuccessful"; mValid = false; - emit downloadDataFinished(nullptr); + emit downloadDataFinished(nullptr); // TODO: think about it, will the other if passed? } else if (!checkResponse(request->responseData())) { qWarning(colnod) << "Data from" << request->url() << "are invalid"; diff --git a/src/colnod/colnodapi.h b/src/colnod/colnodapi.h index 68ed360..f96d2f0 100644 --- a/src/colnod/colnodapi.h +++ b/src/colnod/colnodapi.h @@ -51,7 +51,7 @@ public: ColnodAPI(serversdk::RequestManager* reqManconst, const bool &dryMode = false, const QString &userName = "spaceti", const QString &password = "spaceti1"); void downloadData(const Timestamp &time); - void processData(const std::shared_ptr &data); + bool processData(const std::shared_ptr &data); // New approach std::shared_ptr requestLogin(); diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index cb537e1..6c2a823 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -49,10 +49,6 @@ bool SyncManager::startSynchronization() qInfo(syncmanager) << equals; qInfo(syncmanager) << "SYNCHRONIZATION BEGIN"; - // TODO: solve this in prepare data - database->initNewTokens(); - - // GETTING DATA FROM DATABASE auto databaseData = database->downloadData(); if (!databaseData) { @@ -82,7 +78,12 @@ bool SyncManager::startSynchronization() return; } - // NOT FINISHED scolnod->processData(preparedData->colnodData); + if (!colnod->processData(preparedData->colnodData)) { + qCritical(syncmanager) << "Not possible to upload data to colnod"; + } + if (!database->processData(preparedData->databaseData)) { + qCritical(syncmanager) << "Not possible to upload data to database"; + } }); return true; diff --git a/src/colnod/syncmanager.h b/src/colnod/syncmanager.h index 0d2719b..7b78bc0 100644 --- a/src/colnod/syncmanager.h +++ b/src/colnod/syncmanager.h @@ -30,7 +30,6 @@ private: private: void initNewToken(std::shared_ptr token); - bool downloadData(); bool prepareData(); bool processData(); void updateLastUpdate(qint64 syncBegin); @@ -55,12 +54,6 @@ private: serversdk::DatabaseManager dbMan; Timestamp mlastSync; - std::shared_ptr colnodData; - std::shared_ptr databaseData; - - std::shared_ptr colnodDataToProcess; - std::shared_ptr databaseDataToProcess; - int dataAmount = 0; bool mDryMode; From 5dea3592e5543dcdc424eec38c36be1a0b13b0e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Mon, 2 Dec 2019 13:26:31 +0100 Subject: [PATCH 09/13] fix problems --- src/colnod/colnodapi.cpp | 2 +- src/colnod/colnodapi.h | 21 ++--- src/colnod/syncmanager.cpp | 103 +++++++++++++++++++++- test/auto/colnod/tst_colnodapi.cpp | 2 +- test/auto/syncmanager/tst_syncmanager.cpp | 7 +- 5 files changed, 116 insertions(+), 19 deletions(-) diff --git a/src/colnod/colnodapi.cpp b/src/colnod/colnodapi.cpp index 279b38f..7fd8be1 100644 --- a/src/colnod/colnodapi.cpp +++ b/src/colnod/colnodapi.cpp @@ -243,7 +243,7 @@ qint64 ColnodAPI::lastSync() const void ColnodAPI::setlastSync(const qint64 &lastSync) { - mLastSync.set_timestamp(lastSync); + mLastSync.setTimestamp(lastSync); } diff --git a/src/colnod/colnodapi.h b/src/colnod/colnodapi.h index f96d2f0..81c1c32 100644 --- a/src/colnod/colnodapi.h +++ b/src/colnod/colnodapi.h @@ -10,31 +10,24 @@ Q_DECLARE_LOGGING_CATEGORY(colnod) class LoginResp : public serversdk::DatabaseTable { Q_OBJECT - TABLE_PROPERTY(QString, username, set_username, "") - TABLE_PROPERTY(QString, password, set_password, "") + TABLE_PROPERTY(QString, username, setUsername, "") + TABLE_PROPERTY(QString, password, setPassword, "") }; class AuthToken : public serversdk::DatabaseTable { Q_OBJECT - TABLE_PROPERTY(QString, id, set_id, QString()) + TABLE_PROPERTY(QString, id, setId, QString()) }; class Timestamp : public serversdk::DatabaseTable { Q_OBJECT - TABLE_PROPERTY(qint64, timestamp, set_timestamp, 0) + TABLE_PROPERTY(qint64, timestamp, setTimestamp, 0) public: Timestamp() = default; - explicit Timestamp(const Timestamp &time) + + void setTimestamp(QDateTime time) { - this->set_timestamp(time.timestamp()); - } - explicit Timestamp(qint64 time) - { - set_timestamp(time); - } - explicit Timestamp(QDateTime time) - { - set_timestamp(time.toMSecsSinceEpoch()); + setTimestamp(time.toMSecsSinceEpoch()); } QString toDateString() diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 6c2a823..d2c7638 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -31,7 +31,7 @@ bool SyncManager::databaseInitialization() } // here -> nothing can fail - mlastSync.set_timestamp(database->getLastSync()); + mlastSync.setTimestamp(database->getLastSync()); colnod = std::make_shared(&reqMan, mDryMode); return true; } @@ -213,6 +213,105 @@ void SyncManager::initNewToken(std::shared_ptr token) qInfo(syncmanager).noquote() << Message::massageTemplate("Inicializing new", "token", "sync manager", token->name()); } +std::unique_ptr SyncManager::prepareDataUsindOld(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData) +{ + Q_UNUSED(databaseData); + Q_UNUSED(colnodData); + + qInfo(syncmanager) << equals; + qInfo(syncmanager) << "PREPARING DATA -> BEGIN"; + qInfo(syncmanager) << dash; + qInfo(syncmanager) << "Resolve conflicts ( changes on same item on both Colnod and DB ):"; + qInfo(syncmanager) << dash; + + + Q_ASSERT(colnodData); + Q_ASSERT(databaseData); + + + // TODO: why copy data? + std::shared_ptr cloneColnodDataToProcess = std::make_shared(*colnodData); + std::shared_ptr cloneDatabaseDataToProcess = std::make_shared(*databaseData); + + QList conflicts; + QList alreadySynced; + + // check modifed tokens from colnod, dont want update token, if it was edited in db after or has been already synced + for (const auto &colnodToken : cloneColnodDataToProcess->updatedTokens) { + if (cloneDatabaseDataToProcess->updatedTokens.contains(colnodToken->id())) { + // updated in db later + if (cloneDatabaseDataToProcess->updatedTokens.value(colnodToken->id()) > colnodToken) { + conflicts.append(colnodToken->id()); + } + continue; + } + auto token = database->getToken(colnodToken->id()); + if ( token ) + // alredy synced ( tokens are same ) + if (token == colnodToken) + alreadySynced.append(colnodToken->id()); + } + // remove tokens which should be not updated + for (const auto &id : conflicts) + cloneColnodDataToProcess->updatedTokens.remove(id); + for (const auto &id : alreadySynced) + cloneColnodDataToProcess->updatedTokens.remove(id); + + qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:"; + qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE TOO"; + qInfo(syncmanager) << tab << tab << alreadySynced.size() << "TOKENS WERE ALREADY SYNCED"; + + + + conflicts.clear(); + + // chech for already deleted ( in previous synced tokens were deleted from colnod, so it come to this sync but we dont want to sync it again + for (const auto &id : cloneColnodDataToProcess->deletedTokens) { + if (database->isAlreadyDeleted(id)) { + conflicts.append(id); + } + } + for (const auto &id : conflicts) { + cloneColnodDataToProcess->deletedTokens.removeOne(id); + } + + qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ALREADY DELETED"; + qInfo(syncmanager) << dash; + + conflicts.clear(); + + // chech modified from db, remove token if it was edited in colnod later + for (const auto &databaseToken : cloneDatabaseDataToProcess->updatedTokens) { + if (cloneColnodDataToProcess->updatedTokens.contains(databaseToken->id())) { + if (databaseToken < cloneColnodDataToProcess->updatedTokens.value(databaseToken->id())) + conflicts.append(databaseToken->id()); + } + else { + // restriction cauesed by colnod rest api + if (databaseToken->description() != "mobile") { + qCritical(syncmanager) << "Token:" << databaseToken->id() << "doesn't have 'mobile' in description, can't be "; + conflicts.append(databaseToken->id()); + } + } + } + for (const auto &id : conflicts) + cloneColnodDataToProcess->updatedTokens.remove(id); + + qInfo(syncmanager) << tab << "-> DATABASE DATA - CHANGES:"; + qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN IN COLNOD TOO"; + qInfo(syncmanager) << dash; + + qInfo(syncmanager) << "PREPARING DATA -> FINISHED"; + qInfo(syncmanager) << equals; + + // Prepare data + auto preparedData = std::make_unique(); + preparedData->colnodData = cloneColnodDataToProcess; + preparedData->databaseData = cloneDatabaseDataToProcess; + + return preparedData; +} + bool SyncManager::downloadData() { bool res = true; @@ -338,7 +437,7 @@ bool SyncManager::processData() void SyncManager::updateLastUpdate(qint64 syncBegin) { Timestamp time; - time.set_timestamp(syncBegin); + time.setTimestamp(syncBegin); colnod->getTokensUpdated(time); database->setLastUpdate(databaseDataToProcess->deletedTokens); connect(colnod.get(), &ColnodAPI::partOfDataReady, [=](const std::shared_ptr &data){ diff --git a/test/auto/colnod/tst_colnodapi.cpp b/test/auto/colnod/tst_colnodapi.cpp index dedd554..24354b5 100644 --- a/test/auto/colnod/tst_colnodapi.cpp +++ b/test/auto/colnod/tst_colnodapi.cpp @@ -79,7 +79,7 @@ void ColnodTest::download_data_test() auto data = qvariant_cast>(spy.at(0).at(0)); QVERIFY(data); - QVERIFY(data->updatedTokens.size() == 0); + QVERIFY(data->updatedTokens.isEmpty()); } } diff --git a/test/auto/syncmanager/tst_syncmanager.cpp b/test/auto/syncmanager/tst_syncmanager.cpp index ff3fbaf..6939fc8 100644 --- a/test/auto/syncmanager/tst_syncmanager.cpp +++ b/test/auto/syncmanager/tst_syncmanager.cpp @@ -23,7 +23,12 @@ void SyncManagerTest::prepareData_test() SyncManager sync(true); auto dbData = std::make_shared(); auto colnodData = std::make_shared(); - QVERIFY2(sync.prepareDataNew(dbData, colnodData), "finish unitest"); + + auto oldPreparedData = sync.prepareDataUsindOld(dbData, colnodData); + auto newPreparedData = sync.prepareDataNew(dbData, colnodData); + + QCOMPARE(newPreparedData->databaseData->updatedTokens.size(), oldPreparedData->databaseData->updatedTokens.size()); + QCOMPARE(newPreparedData->databaseData->updatedTokens.size(), oldPreparedData->databaseData->updatedTokens.size()); } QTEST_MAIN(SyncManagerTest) From 7ac0df57e5d16628534acf1e298406b412d4259a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Mon, 2 Dec 2019 13:35:32 +0100 Subject: [PATCH 10/13] continue with fixing --- src/colnod/colnodapi.cpp | 19 +++++++++++-------- src/colnod/colnodapi.h | 4 ++-- src/colnod/syncmanager.cpp | 8 ++++---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/colnod/colnodapi.cpp b/src/colnod/colnodapi.cpp index 7fd8be1..5249f5f 100644 --- a/src/colnod/colnodapi.cpp +++ b/src/colnod/colnodapi.cpp @@ -18,11 +18,13 @@ ColnodAPI::ColnodAPI(serversdk::RequestManager* reqMan, const bool &dryMode, con * @return signal with nullptr when something went wrong * @return signal with data when valid data returned */ -void ColnodAPI::downloadData(const Timestamp &time) +void ColnodAPI::downloadData(const qint64 &time) { // Have to get Login + Timestamp timestamp; + timestamp.setTimestamp(time); auto login = requestLogin(); - connect(login.get(), &serversdk::Request::finished, this, [=](){ + connect(login.get(), &serversdk::Request::finished, this, [&](){ // Get login token if (!checkRequest(login)) { emit downloadDataFinished(nullptr); @@ -47,31 +49,32 @@ void ColnodAPI::downloadData(const Timestamp &time) } }; + // if I have all data emit downloadDataFinished connect(this, &ColnodAPI::partOfDataReady, this, &ColnodAPI::areDataReady); - auto tokensRequest = getTokensUpdated(time); + auto tokensRequest = getTokensUpdated(timestamp); connect(tokensRequest.get(), &serversdk::Request::finished, errorHandler); connect(tokensRequest.get(), &serversdk::Request::finished, [=](){ saveUpdatedTokens(data, tokensRequest->responseData()); emit partOfDataReady(data); }); - auto deletedTokensRequest = getTokensDeleted(time); + auto deletedTokensRequest = getTokensDeleted(timestamp); connect(deletedTokensRequest.get(), &serversdk::Request::finished, errorHandler); connect(deletedTokensRequest.get(), &serversdk::Request::finished, [=](){ saveDeletedTokens(data, deletedTokensRequest->responseData()); emit partOfDataReady(data); }); - auto subjectsRequest = getSubjectsUpdated(time); + auto subjectsRequest = getSubjectsUpdated(timestamp); connect(subjectsRequest.get(), &serversdk::Request::finished, errorHandler); connect(subjectsRequest.get(), &serversdk::Request::finished, [=](){ saveUpdatedSubjects(data, subjectsRequest->responseData()); emit partOfDataReady(data); }); - auto deletedSubjectsRequest = getSubjectsDeleted(time); + auto deletedSubjectsRequest = getSubjectsDeleted(timestamp); connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, errorHandler); connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, [=](){ saveDeletedSubjects(data, deletedSubjectsRequest->responseData()); @@ -114,14 +117,14 @@ std::shared_ptr ColnodAPI::requestLogin() { Q_ASSERT(mReqMan); LoginResp login; - login.set_username(userName()); + login.setUsername(userName()); // password QString encodedPassword = QCryptographicHash::hash( password().toLocal8Bit(), // -> bytearray QCryptographicHash::Sha512 // SHA 512 ).toHex().toUpper(); - login.set_password(encodedPassword); + login.setPassword(encodedPassword); auto loginRequest = composeRequest("/AaaManager/authSession", login.jsonData()); loginRequest->setUserName(login.username()); diff --git a/src/colnod/colnodapi.h b/src/colnod/colnodapi.h index 81c1c32..6f87e11 100644 --- a/src/colnod/colnodapi.h +++ b/src/colnod/colnodapi.h @@ -43,8 +43,8 @@ public: ColnodAPI() = default; ColnodAPI(serversdk::RequestManager* reqManconst, const bool &dryMode = false, const QString &userName = "spaceti", const QString &password = "spaceti1"); - void downloadData(const Timestamp &time); - bool processData(const std::shared_ptr &data); + void downloadData(const qint64 &time); + void processData(const std::shared_ptr &data); // New approach std::shared_ptr requestLogin(); diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index d2c7638..a1f172f 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -58,7 +58,7 @@ bool SyncManager::startSynchronization() // NOW I HAVE DATA FROM DATABASE - colnod->downloadData(Timestamp(syncBegin)); + colnod->downloadData(syncBegin); connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr colnodData) { if (!colnodData) { qCritical(syncmanager) << "Not possible to get correct data from colnod"; @@ -78,9 +78,9 @@ bool SyncManager::startSynchronization() return; } - if (!colnod->processData(preparedData->colnodData)) { - qCritical(syncmanager) << "Not possible to upload data to colnod"; - } + colnod->processData(preparedData->colnodData); + qCritical(syncmanager) << "Not possible to upload data to colnod"; + if (!database->processData(preparedData->databaseData)) { qCritical(syncmanager) << "Not possible to upload data to database"; } From 26fbb6b281f2828776106c9ba687e5ddd73d63e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Mon, 2 Dec 2019 14:30:43 +0100 Subject: [PATCH 11/13] finish main function --- src/colnod/databaseapi.cpp | 12 +- src/colnod/databaseapi.h | 4 +- src/colnod/syncmanager.cpp | 247 +++++++---------------------- src/colnod/syncmanager.h | 14 +- test/auto/colnod/tst_colnodapi.cpp | 16 +- 5 files changed, 67 insertions(+), 226 deletions(-) diff --git a/src/colnod/databaseapi.cpp b/src/colnod/databaseapi.cpp index 7c4b695..05928d7 100644 --- a/src/colnod/databaseapi.cpp +++ b/src/colnod/databaseapi.cpp @@ -118,7 +118,7 @@ void DatabaseAPI::setLastSync(qint64 timestamp) } } -void DatabaseAPI::setLastUpdate(const QHash> &tokens) +void DatabaseAPI::reupdateUpdated(const QHash> &tokens) { auto query = mDbMan->createQuery(); for (const auto &token : tokens){ @@ -135,7 +135,7 @@ void DatabaseAPI::setLastUpdate(const QHash> &to qInfo(database) << equals; } -void DatabaseAPI::setLastUpdate(const QList ids) +void DatabaseAPI::reupdateDeleted(const QList ids) { auto query = mDbMan->createQuery(); for (const auto &id : ids){ @@ -193,14 +193,6 @@ bool DatabaseAPI::uploadTokens(const QHash> &tok for (const auto &token : tokens) { // Get token - QSqlQuery query; - - - - - - - if (exists(token->id(), "colnod_token")) { query = utils->updateTable(token.get(), "colnod_token"); qInfo(database).noquote() << Message::massageTemplate("Updating", "token", "database", token->name()); diff --git a/src/colnod/databaseapi.h b/src/colnod/databaseapi.h index 8668b2f..9a0efeb 100644 --- a/src/colnod/databaseapi.h +++ b/src/colnod/databaseapi.h @@ -31,8 +31,8 @@ public: bool processData(const std::shared_ptr &data); qint64 getLastSync(); void setLastSync(qint64 timestamp); - void setLastUpdate(const QHash > &tokens); - void setLastUpdate(const QList ids); + void reupdateUpdated(const QHash > &tokens); + void reupdateDeleted(const QList ids); qint64 getNow(); void initNewTokens(); diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index a1f172f..5194f23 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -13,9 +13,6 @@ SyncManager::SyncManager(bool dryMode) bool SyncManager::databaseInitialization() { database = std::make_shared(&dbMan, mDryMode); -// connect(database.get(), &DatabaseAPI::error, [=](){ -// QCoreApplication::exit(1); -// }); // Open database if (!database->open()) { @@ -49,6 +46,12 @@ bool SyncManager::startSynchronization() qInfo(syncmanager) << equals; qInfo(syncmanager) << "SYNCHRONIZATION BEGIN"; + qInfo(syncmanager) << equals; + qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN"; + qInfo(syncmanager) << dash; + qInfo(syncmanager).noquote() << "Modified data since last synchronization (" << mlastSync.toDateString() << "):"; + qInfo(syncmanager) << dash; + // GETTING DATA FROM DATABASE auto databaseData = database->downloadData(); if (!databaseData) { @@ -57,9 +60,11 @@ bool SyncManager::startSynchronization() } // NOW I HAVE DATA FROM DATABASE - colnod->downloadData(syncBegin); connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr colnodData) { + qInfo(syncmanager) << "DOWNLOADING DATA -> FINISHED"; + qInfo(syncmanager) << equals; + if (!colnodData) { qCritical(syncmanager) << "Not possible to get correct data from colnod"; QCoreApplication::exit(-1); @@ -67,9 +72,7 @@ bool SyncManager::startSynchronization() } // NOW I SHOULD HAVE VALID DATA FROM COLNOD - // I can process data - qDebug(syncmanager) << "I have colnod and database data"; - qDebug(syncmanager) << "updated token count" << colnodData->updatedTokens.count(); + std::unique_ptr preparedData = prepareDataNew(databaseData, colnodData); if (!preparedData) { @@ -78,51 +81,65 @@ bool SyncManager::startSynchronization() return; } - colnod->processData(preparedData->colnodData); - qCritical(syncmanager) << "Not possible to upload data to colnod"; + qInfo(syncmanager) << equals; + qInfo(syncmanager) << "PROCESSING DATA -> BEGIN"; + qInfo(syncmanager) << dash; + qInfo(syncmanager) << "Push changes to both Colnod and Database:"; + qInfo(syncmanager) << dash; if (!database->processData(preparedData->databaseData)) { qCritical(syncmanager) << "Not possible to upload data to database"; } - }); - - return true; - //downloadData(); - - //connect(this, &SyncManager::dataReady, processDownloadedData); - - - connect(this, &SyncManager::dataReady, [=](){ - prepareData(); - - connect(this, &SyncManager::dataProcessFinished, [=](){ + colnod->processData(preparedData->colnodData); + //qCritical(syncmanager) << "Not possible to upload data to colnod"; + // data is processed to other sides + connect(colnod.get(), &ColnodAPI::dataProcessFinished, [&](){ qInfo(syncmanager) << "PROCESSING DATA -> FINISHED"; qInfo(syncmanager) << equals; - updateLastUpdate(syncBegin); // TODO: change to update all attrinutes - connect(this, &SyncManager::syncFinished, [=](){ - if (database->isValid() && colnod->isValid()) { - database->setLastSync(syncBegin); + // REUPDATE tokens pushed from database to colnod (cause of lastUpdate time) + Timestamp time; + time.setTimestamp(syncBegin); + + database->reupdateDeleted(preparedData->databaseData->deletedTokens); + // download data changes in this sync + colnod->getTokensUpdated(time); + + // updated data are ready + connect(colnod.get(), &ColnodAPI::partOfDataReady, [=](const std::shared_ptr &uploadedData){ + if (!uploadedData) { + qCritical(syncmanager) << "Not possible to get correct data from colnod"; + QCoreApplication::exit(-1); + return; } - else { + if (!database->uploadTokens(uploadedData->updatedTokens)) { + qCritical(syncmanager) << "Not possible to upload data to database"; + } + database->reupdateDeleted(uploadedData->deletedTokens); + + if (!database->isValid() || !colnod->isValid()) { qWarning(syncmanager) << "Synchronization was unsuccessful, next run of sync will be with same last sync time, check warnings"; } + else { + database->setLastSync(syncBegin); + qInfo(syncmanager).noquote() << "Update last synchronization time to time of this sync (" + << QDateTime::fromMSecsSinceEpoch(syncBegin).toString("yyyy-MM-dd hh:mm:ss.zzz") + << ")"; + } - qInfo(syncmanager).noquote() << "Update last synchronization time to time of this sync (" - << QDateTime::fromMSecsSinceEpoch(syncBegin).toString("yyyy-MM-dd hh:mm:ss.zzz") - << ")"; qInfo(syncmanager) << equals; qInfo(syncmanager) << "SYNCHRONIZATION FINISHED"; qInfo(syncmanager) << equals; QCoreApplication::exit(); }); - }); - processData(); + }); }); + return true; + } std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData) @@ -133,6 +150,13 @@ std::unique_ptr SyncManager::prepareDataNew(const std::shared_ptr< QList conflicts; QList alreadySynced; + qInfo(syncmanager) << equals; + qInfo(syncmanager) << "PREPARING DATA -> BEGIN"; + qInfo(syncmanager) << dash; + qInfo(syncmanager) << "Resolve conflicts ( changes on same item on both Colnod and DB ):"; + qInfo(syncmanager) << dash; + + for (const auto &colnodToken : colnodData->updatedTokens) { // not conflict so add to prepered tokens @@ -215,8 +239,8 @@ void SyncManager::initNewToken(std::shared_ptr token) std::unique_ptr SyncManager::prepareDataUsindOld(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData) { - Q_UNUSED(databaseData); - Q_UNUSED(colnodData); + Q_UNUSED(databaseData) + Q_UNUSED(colnodData) qInfo(syncmanager) << equals; qInfo(syncmanager) << "PREPARING DATA -> BEGIN"; @@ -311,160 +335,3 @@ std::unique_ptr SyncManager::prepareDataUsindOld(const std::shared return preparedData; } - -bool SyncManager::downloadData() -{ - bool res = true; - qInfo(syncmanager) << equals; - qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN"; - qInfo(syncmanager) << dash; - qInfo(syncmanager).noquote() << "Modified data since last synchronization (" << mlastSync.toDateString() << "):"; - qInfo(syncmanager) << dash; - - connect(colnod.get(), &ColnodAPI::downloadDataFinished, this, &SyncManager::saveColnodData); - //connect(database.get(), &DatabaseAPI::downloadDataFinished, this, &SyncManager::saveDatabaseData); - //database->downloadData(); - //colnod->downloadData(); - - return res; -} - -// finds conflicts -> changes on both sides (colnod and DB), apply newer change -bool SyncManager::prepareData() -{ - qInfo(syncmanager) << equals; - qInfo(syncmanager) << "PREPARING DATA -> BEGIN"; - qInfo(syncmanager) << dash; - qInfo(syncmanager) << "Resolve conflicts ( changes on same item on both Colnod and DB ):"; - qInfo(syncmanager) << dash; - - - Q_ASSERT(colnodData); - Q_ASSERT(databaseData); - - - // TODO: why copy data? - colnodDataToProcess = std::make_shared(*colnodData); - databaseDataToProcess = std::make_shared(*databaseData); - - QList conflicts; - QList alreadySynced; - - // check modifed tokens from colnod, dont want update token, if it was edited in db after or has been already synced - for (const auto &colnodToken : colnodDataToProcess->updatedTokens) { - if (databaseDataToProcess->updatedTokens.contains(colnodToken->id())) { - // updated in db later - if (databaseDataToProcess->updatedTokens.value(colnodToken->id()) > colnodToken) { - conflicts.append(colnodToken->id()); - } - continue; - } - auto token = database->getToken(colnodToken->id()); - if ( token ) - // alredy synced ( tokens are same ) - if (token == colnodToken) - alreadySynced.append(colnodToken->id()); - } - // remove tokens which should be not updated - for (const auto &id : conflicts) - colnodDataToProcess->updatedTokens.remove(id); - for (const auto &id : alreadySynced) - colnodDataToProcess->updatedTokens.remove(id); - - qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:"; - qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE TOO"; - qInfo(syncmanager) << tab << tab << alreadySynced.size() << "TOKENS WERE ALREADY SYNCED"; - - - - conflicts.clear(); - - // chech for already deleted ( in previous synced tokens were deleted from colnod, so it come to this sync but we dont want to sync it again - for (const auto &id : colnodDataToProcess->deletedTokens) { - if (database->isAlreadyDeleted(id)) { - conflicts.append(id); - } - } - for (const auto &id : conflicts) { - colnodDataToProcess->deletedTokens.removeOne(id); - } - - qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ALREADY DELETED"; - qInfo(syncmanager) << dash; - - conflicts.clear(); - - // chech modified from db, remove token if it was edited in colnod later - for (const auto &databaseToken : databaseDataToProcess->updatedTokens) { - if (colnodDataToProcess->updatedTokens.contains(databaseToken->id())) { - if (databaseToken < colnodDataToProcess->updatedTokens.value(databaseToken->id())) - conflicts.append(databaseToken->id()); - } - else { - // restriction cauesed by colnod rest api - if (databaseToken->description() != "mobile") { - qCritical(syncmanager) << "Token:" << databaseToken->id() << "doesn't have 'mobile' in description, can't be "; - conflicts.append(databaseToken->id()); - } - } - } - for (const auto &id : conflicts) - colnodDataToProcess->updatedTokens.remove(id); - - qInfo(syncmanager) << tab << "-> DATABASE DATA - CHANGES:"; - qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN IN COLNOD TOO"; - qInfo(syncmanager) << dash; - - qInfo(syncmanager) << "PREPARING DATA -> FINISHED"; - qInfo(syncmanager) << equals; - return true; -} - -bool SyncManager::processData() -{ - qInfo(syncmanager) << equals; - qInfo(syncmanager) << "PROCESSING DATA -> BEGIN"; - qInfo(syncmanager) << dash; - qInfo(syncmanager) << "Push changes to both Colnod and Database:"; - qInfo(syncmanager) << dash; - - database->processData(colnodDataToProcess); - connect(colnod.get(), &ColnodAPI::dataProcessFinished, this, &SyncManager::dataProcessFinished); - colnod->processData(databaseDataToProcess); - return true; -} - -void SyncManager::updateLastUpdate(qint64 syncBegin) -{ - Timestamp time; - time.setTimestamp(syncBegin); - colnod->getTokensUpdated(time); - database->setLastUpdate(databaseDataToProcess->deletedTokens); - connect(colnod.get(), &ColnodAPI::partOfDataReady, [=](const std::shared_ptr &data){ - database->setLastUpdate(data->updatedTokens); - emit syncFinished(); - }); -} - -void SyncManager::areDataReady() -{ - if (++dataAmount == 1) { - - qInfo(syncmanager) << "DOWNLOADING DATA -> FINISHED"; - qInfo(syncmanager) << equals; - - emit dataReady(); - } -} - -void SyncManager::saveColnodData(const std::shared_ptr &data) -{ - colnodData = data; - areDataReady(); -} - -void SyncManager::saveDatabaseData(const std::shared_ptr &data) -{ - databaseData = data; - areDataReady(); -} diff --git a/src/colnod/syncmanager.h b/src/colnod/syncmanager.h index 7b78bc0..93e16e7 100644 --- a/src/colnod/syncmanager.h +++ b/src/colnod/syncmanager.h @@ -26,23 +26,14 @@ private: * @todo rename when removed old aproach */ std::unique_ptr prepareDataNew(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData); + std::unique_ptr prepareDataUsindOld(const std::shared_ptr &databaseData, const std::shared_ptr &colnodData); private: void initNewToken(std::shared_ptr token); - bool prepareData(); - bool processData(); void updateLastUpdate(qint64 syncBegin); - -private: - Q_SLOT void saveColnodData(const std::shared_ptr &data); - Q_SLOT void saveDatabaseData(const std::shared_ptr &data); - Q_SLOT void areDataReady(); - signals: - void dataReady(); - void dataProcessFinished(); void syncFinished(); void error(); @@ -54,8 +45,5 @@ private: serversdk::DatabaseManager dbMan; Timestamp mlastSync; - int dataAmount = 0; bool mDryMode; - - std::function onError = nullptr; }; diff --git a/test/auto/colnod/tst_colnodapi.cpp b/test/auto/colnod/tst_colnodapi.cpp index 24354b5..7e6142c 100644 --- a/test/auto/colnod/tst_colnodapi.cpp +++ b/test/auto/colnod/tst_colnodapi.cpp @@ -27,9 +27,6 @@ private: QString getLoginToken(ColnodAPI &colnod); }; - - - void ColnodTest::login() { serversdk::RequestManager reqMan; @@ -52,7 +49,8 @@ void ColnodTest::getTokens() QSignalSpy spy(login.get(), &serversdk::Request::finished); spy.wait(); QString token = getLoginToken(colnod); - Timestamp time(QDateTime::currentMSecsSinceEpoch()); + Timestamp time; + time.setTimestamp(QDateTime::currentMSecsSinceEpoch()); colnod.getTokensUpdated(time); } @@ -62,7 +60,7 @@ void ColnodTest::download_data_test() ColnodAPI colnod(&reqMan); { QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished); - colnod.downloadData(Timestamp(1570190399259)); + colnod.downloadData(1570190399259); QVERIFY(spy.wait()); auto data = qvariant_cast>(spy.at(0).at(0)); @@ -72,8 +70,7 @@ void ColnodTest::download_data_test() QVERIFY(data->updatedTokens.contains("5e2f1260-da1c-441f-9644-24dfe3b97b2a")); } { - Timestamp time(QDateTime::currentMSecsSinceEpoch()); - colnod.downloadData(time); + colnod.downloadData(QDateTime::currentMSecsSinceEpoch()); QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished); QVERIFY(spy.wait()); auto data = qvariant_cast>(spy.at(0).at(0)); @@ -88,17 +85,14 @@ void ColnodTest::download_data_fail_test() serversdk::RequestManager reqMan; ColnodAPI colnod(&reqMan, false, "spaceti", "wrong password"); - Timestamp time(0); QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished); - colnod.downloadData(time); + colnod.downloadData(0); QVERIFY(spy.wait()); auto data = qvariant_cast>(spy.at(0).at(0)); QVERIFY(!data); - - } QString ColnodTest::getLoginToken(ColnodAPI &colnod) From fd4440f9408eb01edb8449a78b7acc083eea3939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Mon, 2 Dec 2019 15:04:51 +0100 Subject: [PATCH 12/13] WIP: doesnt work (stacked in event loop) --- src/colnod/colnodapi.cpp | 8 ++++---- src/colnod/syncmanager.cpp | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/colnod/colnodapi.cpp b/src/colnod/colnodapi.cpp index 5249f5f..de04fc5 100644 --- a/src/colnod/colnodapi.cpp +++ b/src/colnod/colnodapi.cpp @@ -21,16 +21,13 @@ ColnodAPI::ColnodAPI(serversdk::RequestManager* reqMan, const bool &dryMode, con void ColnodAPI::downloadData(const qint64 &time) { // Have to get Login - Timestamp timestamp; - timestamp.setTimestamp(time); auto login = requestLogin(); - connect(login.get(), &serversdk::Request::finished, this, [&](){ + connect(login.get(), &serversdk::Request::finished, this, [=](){ // Get login token if (!checkRequest(login)) { emit downloadDataFinished(nullptr); return; } - AuthToken token; token.fromJson(login->responseData()); authToken = token.id(); @@ -53,6 +50,9 @@ void ColnodAPI::downloadData(const qint64 &time) // if I have all data emit downloadDataFinished connect(this, &ColnodAPI::partOfDataReady, this, &ColnodAPI::areDataReady); + Timestamp timestamp; + timestamp.setTimestamp(time); + auto tokensRequest = getTokensUpdated(timestamp); connect(tokensRequest.get(), &serversdk::Request::finished, errorHandler); connect(tokensRequest.get(), &serversdk::Request::finished, [=](){ diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 5194f23..2d01b56 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -90,8 +90,6 @@ bool SyncManager::startSynchronization() if (!database->processData(preparedData->databaseData)) { qCritical(syncmanager) << "Not possible to upload data to database"; } - colnod->processData(preparedData->colnodData); - //qCritical(syncmanager) << "Not possible to upload data to colnod"; // data is processed to other sides connect(colnod.get(), &ColnodAPI::dataProcessFinished, [&](){ @@ -135,6 +133,9 @@ bool SyncManager::startSynchronization() QCoreApplication::exit(); }); + colnod->processData(preparedData->colnodData); + //qCritical(syncmanager) << "Not possible to upload data to colnod"; + }); }); From cca625ddec2d60223eee273b4935e79cfaec8b67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Mon, 2 Dec 2019 15:45:21 +0100 Subject: [PATCH 13/13] fix wrong paste --- src/colnod/syncmanager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/colnod/syncmanager.cpp b/src/colnod/syncmanager.cpp index 2d01b56..a1ba4e9 100644 --- a/src/colnod/syncmanager.cpp +++ b/src/colnod/syncmanager.cpp @@ -133,10 +133,10 @@ bool SyncManager::startSynchronization() QCoreApplication::exit(); }); - colnod->processData(preparedData->colnodData); - //qCritical(syncmanager) << "Not possible to upload data to colnod"; }); + colnod->processData(preparedData->colnodData); + //qCritical(syncmanager) << "Not possible to upload data to colnod"; }); return true;