ondra/colnod-connector
1
0
Fork 1
mirror of https://gitlab.com/spaceti-app/integrations/acs/colnod-connector.git synced 2026-07-12 13:10:45 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'suggest/process-data' into 'master'

Browse Source 
Suggest/process data

See merge request spaceti-app/integrations/acs/colnod-connector!32
This commit is contained in:
Filip Bucek 2019-12-02 15:02:32 +00:00
commit d1a9c56f62
9 changed files with 257 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/colnod/colnodapi.cpp
View File

@ -11,7 +11,14 @@ ColnodAPI::ColnodAPI(serversdk::RequestManager* reqMan, const bool &dryMode, con
// ----------------------------------- // -----------------------------------
} }
void ColnodAPI::downloadData(const Timestamp &time) /**
* @brief Downloads new data from Colnod -> emits downloadDataFinished signal
* @param time
*
* @return signal with nullptr when something went wrong
* @return signal with data when valid data returned
*/
void ColnodAPI::downloadData(const qint64 &time)
{ {
// Have to get Login // Have to get Login
auto login = requestLogin(); auto login = requestLogin();
@ -21,7 +28,6 @@ void ColnodAPI::downloadData(const Timestamp &time)
emit downloadDataFinished(nullptr); emit downloadDataFinished(nullptr);
return; return;
} }
AuthToken token; AuthToken token;
token.fromJson(login->responseData()); token.fromJson(login->responseData());
authToken = token.id(); authToken = token.id();
@ -32,7 +38,7 @@ void ColnodAPI::downloadData(const Timestamp &time)
if (!checkRequest(request)) { if (!checkRequest(request)) {
qWarning(colnod) << "ERROR during calling" << request->url() << ", request unsuccessful"; qWarning(colnod) << "ERROR during calling" << request->url() << ", request unsuccessful";
mValid = false; mValid = false;
emit downloadDataFinished(nullptr); emit downloadDataFinished(nullptr); // TODO: think about it, will the other if passed?
} }
else if (!checkResponse(request->responseData())) { else if (!checkResponse(request->responseData())) {
qWarning(colnod) << "Data from" << request->url() << "are invalid"; qWarning(colnod) << "Data from" << request->url() << "are invalid";
@ -40,31 +46,35 @@ void ColnodAPI::downloadData(const Timestamp &time)
} }
}; };
// if I have all data emit downloadDataFinished // if I have all data emit downloadDataFinished
connect(this, &ColnodAPI::partOfDataReady, this, &ColnodAPI::areDataReady); connect(this, &ColnodAPI::partOfDataReady, this, &ColnodAPI::areDataReady);
auto tokensRequest = getTokensUpdated(time); Timestamp timestamp;
timestamp.setTimestamp(time);
auto tokensRequest = getTokensUpdated(timestamp);
connect(tokensRequest.get(), &serversdk::Request::finished, errorHandler); connect(tokensRequest.get(), &serversdk::Request::finished, errorHandler);
connect(tokensRequest.get(), &serversdk::Request::finished, [=](){ connect(tokensRequest.get(), &serversdk::Request::finished, [=](){
saveUpdatedTokens(data, tokensRequest->responseData()); saveUpdatedTokens(data, tokensRequest->responseData());
emit partOfDataReady(data); emit partOfDataReady(data);
}); });
auto deletedTokensRequest = getTokensDeleted(time); auto deletedTokensRequest = getTokensDeleted(timestamp);
connect(deletedTokensRequest.get(), &serversdk::Request::finished, errorHandler); connect(deletedTokensRequest.get(), &serversdk::Request::finished, errorHandler);
connect(deletedTokensRequest.get(), &serversdk::Request::finished, [=](){ connect(deletedTokensRequest.get(), &serversdk::Request::finished, [=](){
saveDeletedTokens(data, deletedTokensRequest->responseData()); saveDeletedTokens(data, deletedTokensRequest->responseData());
emit partOfDataReady(data); emit partOfDataReady(data);
}); });
auto subjectsRequest = getSubjectsUpdated(time); auto subjectsRequest = getSubjectsUpdated(timestamp);
connect(subjectsRequest.get(), &serversdk::Request::finished, errorHandler); connect(subjectsRequest.get(), &serversdk::Request::finished, errorHandler);
connect(subjectsRequest.get(), &serversdk::Request::finished, [=](){ connect(subjectsRequest.get(), &serversdk::Request::finished, [=](){
saveUpdatedSubjects(data, subjectsRequest->responseData()); saveUpdatedSubjects(data, subjectsRequest->responseData());
emit partOfDataReady(data); emit partOfDataReady(data);
}); });
auto deletedSubjectsRequest = getSubjectsDeleted(time); auto deletedSubjectsRequest = getSubjectsDeleted(timestamp);
connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, errorHandler); connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, errorHandler);
connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, [=](){ connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, [=](){
saveDeletedSubjects(data, deletedSubjectsRequest->responseData()); saveDeletedSubjects(data, deletedSubjectsRequest->responseData());
@ -107,14 +117,14 @@ std::shared_ptr<serversdk::Request> ColnodAPI::requestLogin()
{ {
Q_ASSERT(mReqMan); Q_ASSERT(mReqMan);
LoginResp login; LoginResp login;
login.set_username(userName()); login.setUsername(userName());
// password // password
QString encodedPassword = QCryptographicHash::hash( QString encodedPassword = QCryptographicHash::hash(
password().toLocal8Bit(), // -> bytearray password().toLocal8Bit(), // -> bytearray
QCryptographicHash::Sha512 // SHA 512 QCryptographicHash::Sha512 // SHA 512
).toHex().toUpper(); ).toHex().toUpper();
login.set_password(encodedPassword); login.setPassword(encodedPassword);
auto loginRequest = composeRequest("/AaaManager/authSession", login.jsonData()); auto loginRequest = composeRequest("/AaaManager/authSession", login.jsonData());
loginRequest->setUserName(login.username()); loginRequest->setUserName(login.username());
@ -236,7 +246,7 @@ qint64 ColnodAPI::lastSync() const
void ColnodAPI::setlastSync(const qint64 &lastSync) void ColnodAPI::setlastSync(const qint64 &lastSync)
{ {
mLastSync.set_timestamp(lastSync); mLastSync.setTimestamp(lastSync);
} }

23
src/colnod/colnodapi.h
View File

@ -10,31 +10,24 @@ Q_DECLARE_LOGGING_CATEGORY(colnod)
class LoginResp : public serversdk::DatabaseTable { class LoginResp : public serversdk::DatabaseTable {
Q_OBJECT Q_OBJECT
TABLE_PROPERTY(QString, username, set_username, "") TABLE_PROPERTY(QString, username, setUsername, "")
TABLE_PROPERTY(QString, password, set_password, "") TABLE_PROPERTY(QString, password, setPassword, "")
}; };
class AuthToken : public serversdk::DatabaseTable { class AuthToken : public serversdk::DatabaseTable {
Q_OBJECT Q_OBJECT
TABLE_PROPERTY(QString, id, set_id, QString()) TABLE_PROPERTY(QString, id, setId, QString())
}; };
class Timestamp : public serversdk::DatabaseTable { class Timestamp : public serversdk::DatabaseTable {
Q_OBJECT Q_OBJECT
TABLE_PROPERTY(qint64, timestamp, set_timestamp, 0) TABLE_PROPERTY(qint64, timestamp, setTimestamp, 0)
public: public:
Timestamp() = default; Timestamp() = default;
explicit Timestamp(const Timestamp &time)
void setTimestamp(QDateTime time)
{ {
this->set_timestamp(time.timestamp()); setTimestamp(time.toMSecsSinceEpoch());
}
explicit Timestamp(qint64 time)
{
set_timestamp(time);
}
explicit Timestamp(QDateTime time)
{
set_timestamp(time.toMSecsSinceEpoch());
} }
QString toDateString() QString toDateString()
@ -50,7 +43,7 @@ public:
ColnodAPI() = default; ColnodAPI() = default;
ColnodAPI(serversdk::RequestManager* reqManconst, const bool &dryMode = false, const QString &userName = "spaceti", const QString &password = "spaceti1"); ColnodAPI(serversdk::RequestManager* reqManconst, const bool &dryMode = false, const QString &userName = "spaceti", const QString &password = "spaceti1");
void downloadData(const Timestamp &time); void downloadData(const qint64 &time);
void processData(const std::shared_ptr<Data> &data); void processData(const std::shared_ptr<Data> &data);
// New approach // New approach

14
src/colnod/data.h
View File

@ -37,3 +37,17 @@ public:
QList<QString> deletedSubjects; QList<QString> deletedSubjects;
}; };
struct PreparedData
{
public:
PreparedData() = default;
PreparedData(const std::shared_ptr<Data> &database, const std::shared_ptr<Data> &colnod)
:databaseData(database), colnodData(colnod)
{
}
// TODO: make unique_ptr ( neeed because of ColnodAPI )
std::shared_ptr<Data> databaseData;
std::shared_ptr<Data> colnodData;
};

12
src/colnod/databaseapi.cpp
View File

@ -118,7 +118,7 @@ void DatabaseAPI::setLastSync(qint64 timestamp)
} }
} }
void DatabaseAPI::setLastUpdate(const QHash<QString, std::shared_ptr<Token>> &tokens) void DatabaseAPI::reupdateUpdated(const QHash<QString, std::shared_ptr<Token>> &tokens)
{ {
auto query = mDbMan->createQuery(); auto query = mDbMan->createQuery();
for (const auto &token : tokens){ for (const auto &token : tokens){
@ -135,7 +135,7 @@ void DatabaseAPI::setLastUpdate(const QHash<QString, std::shared_ptr<Token>> &to
qInfo(database) << equals; qInfo(database) << equals;
} }
void DatabaseAPI::setLastUpdate(const QList<QString> ids) void DatabaseAPI::reupdateDeleted(const QList<QString> ids)
{ {
auto query = mDbMan->createQuery(); auto query = mDbMan->createQuery();
for (const auto &id : ids){ for (const auto &id : ids){
@ -193,14 +193,6 @@ bool DatabaseAPI::uploadTokens(const QHash<QString, std::shared_ptr<Token>> &tok
for (const auto &token : tokens) { for (const auto &token : tokens) {
// Get token // Get token
QSqlQuery query;
if (exists(token->id(), "colnod_token")) { if (exists(token->id(), "colnod_token")) {
query = utils->updateTable(token.get(), "colnod_token"); query = utils->updateTable(token.get(), "colnod_token");
qInfo(database).noquote() << Message::massageTemplate("Updating", "token", "database", token->name()); qInfo(database).noquote() << Message::massageTemplate("Updating", "token", "database", token->name());

9
src/colnod/databaseapi.h
View File

@ -31,8 +31,8 @@ public:
bool processData(const std::shared_ptr<Data> &data); bool processData(const std::shared_ptr<Data> &data);
qint64 getLastSync(); qint64 getLastSync();
void setLastSync(qint64 timestamp); void setLastSync(qint64 timestamp);
void setLastUpdate(const QHash<QString, std::shared_ptr<Token> > &tokens); void reupdateUpdated(const QHash<QString, std::shared_ptr<Token> > &tokens);
void setLastUpdate(const QList<QString> ids); void reupdateDeleted(const QList<QString> ids);
qint64 getNow(); qint64 getNow();
void initNewTokens(); void initNewTokens();
@ -48,13 +48,14 @@ public:
bool isValid(); bool isValid();
qint16 getNumOfTokensOfSubject(const QString &id);
QString getSubjectNameByTokenId(const QString &tokenId);
private: private:
int exists(const QString &id, const QString &tableName); int exists(const QString &id, const QString &tableName);
QSqlQuery getUpdatedTokens(); QSqlQuery getUpdatedTokens();
std::shared_ptr<Data> composeData(const QSqlQuery &updatedQuery, QSqlQuery &deletedQuery); std::shared_ptr<Data> composeData(const QSqlQuery &updatedQuery, QSqlQuery &deletedQuery);
qint16 getNumOfTokensOfSubject(const QString &id);
QString getSubjectNameByTokenId(const QString &tokenId);
signals: signals:
void downloadDataFinished(std::shared_ptr<Data> = std::shared_ptr<Data>()); void downloadDataFinished(std::shared_ptr<Data> = std::shared_ptr<Data>());

281
src/colnod/syncmanager.cpp
View File

@ -13,9 +13,6 @@ SyncManager::SyncManager(bool dryMode)
bool SyncManager::databaseInitialization() bool SyncManager::databaseInitialization()
{ {
database = std::make_shared<DatabaseAPI>(&dbMan, mDryMode); database = std::make_shared<DatabaseAPI>(&dbMan, mDryMode);
// connect(database.get(), &DatabaseAPI::error, [=](){
// QCoreApplication::exit(1);
// });
// Open database // Open database
if (!database->open()) { if (!database->open()) {
@ -31,7 +28,7 @@ bool SyncManager::databaseInitialization()
} }
// here -> nothing can fail // here -> nothing can fail
mlastSync.set_timestamp(database->getLastSync()); mlastSync.setTimestamp(database->getLastSync());
colnod = std::make_shared<ColnodAPI>(&reqMan, mDryMode); colnod = std::make_shared<ColnodAPI>(&reqMan, mDryMode);
return true; return true;
} }
@ -49,9 +46,11 @@ bool SyncManager::startSynchronization()
qInfo(syncmanager) << equals; qInfo(syncmanager) << equals;
qInfo(syncmanager) << "SYNCHRONIZATION BEGIN"; qInfo(syncmanager) << "SYNCHRONIZATION BEGIN";
// TODO: check error qInfo(syncmanager) << equals;
database->initNewTokens(); qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN";
qInfo(syncmanager) << dash;
qInfo(syncmanager).noquote() << "Modified data since last synchronization (" << mlastSync.toDateString() << "):";
qInfo(syncmanager) << dash;
// GETTING DATA FROM DATABASE // GETTING DATA FROM DATABASE
auto databaseData = database->downloadData(); auto databaseData = database->downloadData();
@ -61,86 +60,189 @@ bool SyncManager::startSynchronization()
} }
// NOW I HAVE DATA FROM DATABASE // NOW I HAVE DATA FROM DATABASE
colnod->downloadData(syncBegin);
connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr<Data> colnodData) {
qInfo(syncmanager) << "DOWNLOADING DATA -> FINISHED";
qInfo(syncmanager) << equals;
colnod->downloadData(Timestamp(syncBegin)); if (!colnodData) {
connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr<Data> data) {
if (!data) {
qCritical(syncmanager) << "Not possible to get correct data from colnod"; qCritical(syncmanager) << "Not possible to get correct data from colnod";
QCoreApplication::exit(-1);
return; return;
} }
// NOW I SHOULD HAVE VALID DATA FROM COLNOD
// NOW I HAVE DATA FROM COLNOD std::unique_ptr<PreparedData> preparedData = prepareDataNew(databaseData, colnodData);
// I can process data if (!preparedData) {
qDebug(syncmanager) << "I have colnod and database data"; qCritical(syncmanager) << "Not possible to get correct data from colnod";
qDebug(syncmanager) << "updated token count" << data->updatedTokens.count(); QCoreApplication::exit(-1);
return;
}
}); qInfo(syncmanager) << equals;
qInfo(syncmanager) << "PROCESSING DATA -> BEGIN";
qInfo(syncmanager) << dash;
qInfo(syncmanager) << "Push changes to both Colnod and Database:";
qInfo(syncmanager) << dash;
return true; if (!database->processData(preparedData->databaseData)) {
//downloadData(); qCritical(syncmanager) << "Not possible to upload data to database";
}
auto colnodDataFinished = [=](std::shared_ptr<Data> data) {
qDebug() << "Now I can check that I have all data I need";
};
//connect(this, &SyncManager::dataReady, processDownloadedData);
connect(this, &SyncManager::dataReady, [=](){
prepareData();
connect(this, &SyncManager::dataProcessFinished, [=](){
// data is processed to other sides
connect(colnod.get(), &ColnodAPI::dataProcessFinished, [&](){
qInfo(syncmanager) << "PROCESSING DATA -> FINISHED"; qInfo(syncmanager) << "PROCESSING DATA -> FINISHED";
qInfo(syncmanager) << equals; qInfo(syncmanager) << equals;
updateLastUpdate(syncBegin); // REUPDATE tokens pushed from database to colnod (cause of lastUpdate time)
connect(this, &SyncManager::syncFinished, [=](){ Timestamp time;
if (database->isValid() && colnod->isValid()) { time.setTimestamp(syncBegin);
database->setLastSync(syncBegin);
database->reupdateDeleted(preparedData->databaseData->deletedTokens);
// download data changes in this sync
colnod->getTokensUpdated(time);
// updated data are ready
connect(colnod.get(), &ColnodAPI::partOfDataReady, [=](const std::shared_ptr<Data> &uploadedData){
if (!uploadedData) {
qCritical(syncmanager) << "Not possible to get correct data from colnod";
QCoreApplication::exit(-1);
return;
} }
else { if (!database->uploadTokens(uploadedData->updatedTokens)) {
qCritical(syncmanager) << "Not possible to upload data to database";
}
database->reupdateDeleted(uploadedData->deletedTokens);
if (!database->isValid() || !colnod->isValid()) {
qWarning(syncmanager) << "Synchronization was unsuccessful, next run of sync will be with same last sync time, check warnings"; qWarning(syncmanager) << "Synchronization was unsuccessful, next run of sync will be with same last sync time, check warnings";
} }
else {
database->setLastSync(syncBegin);
qInfo(syncmanager).noquote() << "Update last synchronization time to time of this sync (" qInfo(syncmanager).noquote() << "Update last synchronization time to time of this sync ("
<< QDateTime::fromMSecsSinceEpoch(syncBegin).toString("yyyy-MM-dd hh:mm:ss.zzz") << QDateTime::fromMSecsSinceEpoch(syncBegin).toString("yyyy-MM-dd hh:mm:ss.zzz")
<< ")"; << ")";
}
qInfo(syncmanager) << equals; qInfo(syncmanager) << equals;
qInfo(syncmanager) << "SYNCHRONIZATION FINISHED"; qInfo(syncmanager) << "SYNCHRONIZATION FINISHED";
qInfo(syncmanager) << equals; qInfo(syncmanager) << equals;
QCoreApplication::exit(); QCoreApplication::exit();
}); });
});
colnod->processData(preparedData->colnodData);
//qCritical(syncmanager) << "Not possible to upload data to colnod";
}); });
processData();
});
return true; return true;
} }
bool SyncManager::downloadData() std::unique_ptr<PreparedData> SyncManager::prepareDataNew(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData)
{ {
bool res = true; auto preparedColnodData = std::make_shared<Data>();
auto preparedDatabaseData = std::make_shared<Data>();
QList<QString> conflicts;
QList<QString> alreadySynced;
qInfo(syncmanager) << equals; qInfo(syncmanager) << equals;
qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN"; qInfo(syncmanager) << "PREPARING DATA -> BEGIN";
qInfo(syncmanager) << dash; qInfo(syncmanager) << dash;
qInfo(syncmanager).noquote() << "Modified data since last synchronization (" << mlastSync.toDateString() << "):"; qInfo(syncmanager) << "Resolve conflicts ( changes on same item on both Colnod and DB ):";
qInfo(syncmanager) << dash; qInfo(syncmanager) << dash;
connect(colnod.get(), &ColnodAPI::downloadDataFinished, this, &SyncManager::saveColnodData);
//connect(database.get(), &DatabaseAPI::downloadDataFinished, this, &SyncManager::saveDatabaseData);
//database->downloadData();
//colnod->downloadData();
return res; for (const auto &colnodToken : colnodData->updatedTokens) {
// not conflict so add to prepered tokens
if (!databaseData->updatedTokens.contains(colnodToken->id())) {
preparedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken);
continue;
} }
// finds conflicts -> changes on both sides (colnod and DB), apply newer change // change in colnod is newer
bool SyncManager::prepareData() if (databaseData->updatedTokens.value(colnodToken->id()) < colnodToken) {
preparedColnodData->updatedTokens.insert(colnodToken->id(), colnodToken);
}
else {
conflicts.append(colnodToken->id());
}
auto token = database->getToken(colnodToken->id());
if ( token )
// alredy synced ( tokens are same )
if (token == colnodToken)
alreadySynced.append(colnodToken->id());
}
for (const auto &id : colnodData->deletedTokens) {
if (!database->isAlreadyDeleted(id)) {
preparedColnodData->deletedTokens.append(id);
}
}
qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:";
qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE AFTER";
qInfo(syncmanager) << tab << tab << alreadySynced.size() << "TOKENS WERE ALREADY SYNCED";
qInfo(syncmanager) << tab << tab << colnodData->deletedTokens.size() - preparedColnodData->deletedTokens.size() << "TOKENS WERE ALREADY DELETED";
qInfo(syncmanager) << dash;
for (const auto &databaseToken : databaseData->updatedTokens) {
if (databaseToken->name().isEmpty()) {
initNewToken(databaseToken);
}
// restriction cauesed by colnod rest api
if (databaseToken->description() != "mobile") {
qCritical(syncmanager) << "Token:" << databaseToken->id() << "doesn't have 'mobile' in description, can't be ";
continue;
}
// no conflict
if (!colnodData->updatedTokens.contains(databaseToken->id())) {
preparedDatabaseData->updatedTokens.insert(databaseToken->id(), databaseToken);
continue;
}
// database change is newer
if (databaseToken > colnodData->updatedTokens.value(databaseToken->id())) {
preparedDatabaseData->updatedTokens.insert(databaseToken->id(), databaseToken);
}
}
qInfo(syncmanager) << tab << "-> DATABASE DATA - CHANGES:";
qInfo(syncmanager) << tab << tab << databaseData->updatedTokens.size() - preparedColnodData->updatedTokens.size() << "TOKENS WERE ADDED/UPDATED IN COLNOD AFTER";
qInfo(syncmanager) << dash;
qInfo(syncmanager) << "PREPARING DATA -> FINISHED";
qInfo(syncmanager) << equals;
auto preparedData = std::make_unique<PreparedData>(preparedDatabaseData, preparedColnodData);
return preparedData;
}
void SyncManager::initNewToken(std::shared_ptr<Token> token)
{ {
qint16 num = database->getNumOfTokensOfSubject(token->subjectId());
token->setName(database->getSubjectNameByTokenId(token->id()) + "_mobile_" + QString::number(num + 1));
token->setDescription("mobile");
token->setTokenData(token->tokenData().toUpper());
qInfo(syncmanager).noquote() << Message::massageTemplate("Inicializing new", "token", "sync manager", token->name());
}
std::unique_ptr<PreparedData> SyncManager::prepareDataUsindOld(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData)
{
Q_UNUSED(databaseData)
Q_UNUSED(colnodData)
qInfo(syncmanager) << equals; qInfo(syncmanager) << equals;
qInfo(syncmanager) << "PREPARING DATA -> BEGIN"; qInfo(syncmanager) << "PREPARING DATA -> BEGIN";
qInfo(syncmanager) << dash; qInfo(syncmanager) << dash;
@ -153,17 +255,17 @@ bool SyncManager::prepareData()
// TODO: why copy data? // TODO: why copy data?
colnodDataToProcess = std::make_shared<Data>(*colnodData); std::shared_ptr<Data> cloneColnodDataToProcess = std::make_shared<Data>(*colnodData);
databaseDataToProcess = std::make_shared<Data>(*databaseData); std::shared_ptr<Data> cloneDatabaseDataToProcess = std::make_shared<Data>(*databaseData);
QList<QString> conflicts; QList<QString> conflicts;
QList<QString> alreadySynced; QList<QString> alreadySynced;
// check modifed tokens from colnod, dont want update token, if it was edited in db after or has been already synced // check modifed tokens from colnod, dont want update token, if it was edited in db after or has been already synced
for (const auto &colnodToken : colnodDataToProcess->updatedTokens) { for (const auto &colnodToken : cloneColnodDataToProcess->updatedTokens) {
if (databaseDataToProcess->updatedTokens.contains(colnodToken->id())) { if (cloneDatabaseDataToProcess->updatedTokens.contains(colnodToken->id())) {
// updated in db later // updated in db later
if (databaseDataToProcess->updatedTokens.value(colnodToken->id()) > colnodToken) { if (cloneDatabaseDataToProcess->updatedTokens.value(colnodToken->id()) > colnodToken) {
conflicts.append(colnodToken->id()); conflicts.append(colnodToken->id());
} }
continue; continue;
@ -176,9 +278,9 @@ bool SyncManager::prepareData()
} }
// remove tokens which should be not updated // remove tokens which should be not updated
for (const auto &id : conflicts) for (const auto &id : conflicts)
colnodDataToProcess->updatedTokens.remove(id); cloneColnodDataToProcess->updatedTokens.remove(id);
for (const auto &id : alreadySynced) for (const auto &id : alreadySynced)
colnodDataToProcess->updatedTokens.remove(id); cloneColnodDataToProcess->updatedTokens.remove(id);
qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:"; qInfo(syncmanager) << tab << "-> COLNOD DATA - CHANGES:";
qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE TOO"; qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN DATABASE TOO";
@ -189,13 +291,13 @@ bool SyncManager::prepareData()
conflicts.clear(); conflicts.clear();
// chech for already deleted ( in previous synced tokens were deleted from colnod, so it come to this sync but we dont want to sync it again // chech for already deleted ( in previous synced tokens were deleted from colnod, so it come to this sync but we dont want to sync it again
for (const auto &id : colnodDataToProcess->deletedTokens) { for (const auto &id : cloneColnodDataToProcess->deletedTokens) {
if (database->isAlreadyDeleted(id)) { if (database->isAlreadyDeleted(id)) {
conflicts.append(id); conflicts.append(id);
} }
} }
for (const auto &id : conflicts) { for (const auto &id : conflicts) {
colnodDataToProcess->deletedTokens.removeOne(id); cloneColnodDataToProcess->deletedTokens.removeOne(id);
} }
qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ALREADY DELETED"; qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ALREADY DELETED";
@ -204,9 +306,9 @@ bool SyncManager::prepareData()
conflicts.clear(); conflicts.clear();
// chech modified from db, remove token if it was edited in colnod later // chech modified from db, remove token if it was edited in colnod later
for (const auto &databaseToken : databaseDataToProcess->updatedTokens) { for (const auto &databaseToken : cloneDatabaseDataToProcess->updatedTokens) {
if (colnodDataToProcess->updatedTokens.contains(databaseToken->id())) { if (cloneColnodDataToProcess->updatedTokens.contains(databaseToken->id())) {
if (databaseToken < colnodDataToProcess->updatedTokens.value(databaseToken->id())) if (databaseToken < cloneColnodDataToProcess->updatedTokens.value(databaseToken->id()))
conflicts.append(databaseToken->id()); conflicts.append(databaseToken->id());
} }
else { else {
@ -218,7 +320,7 @@ bool SyncManager::prepareData()
} }
} }
for (const auto &id : conflicts) for (const auto &id : conflicts)
colnodDataToProcess->updatedTokens.remove(id); cloneColnodDataToProcess->updatedTokens.remove(id);
qInfo(syncmanager) << tab << "-> DATABASE DATA - CHANGES:"; qInfo(syncmanager) << tab << "-> DATABASE DATA - CHANGES:";
qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN IN COLNOD TOO"; qInfo(syncmanager) << tab << tab << conflicts.size() << "TOKENS WERE ADDED/UPDATED IN IN COLNOD TOO";
@ -226,54 +328,11 @@ bool SyncManager::prepareData()
qInfo(syncmanager) << "PREPARING DATA -> FINISHED"; qInfo(syncmanager) << "PREPARING DATA -> FINISHED";
qInfo(syncmanager) << equals; qInfo(syncmanager) << equals;
return true;
} // Prepare data
auto preparedData = std::make_unique<PreparedData>();
bool SyncManager::processData() preparedData->colnodData = cloneColnodDataToProcess;
{ preparedData->databaseData = cloneDatabaseDataToProcess;
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "PROCESSING DATA -> BEGIN"; return preparedData;
qInfo(syncmanager) << dash;
qInfo(syncmanager) << "Push changes to both Colnod and Database:";
qInfo(syncmanager) << dash;
database->processData(colnodDataToProcess);
connect(colnod.get(), &ColnodAPI::dataProcessFinished, this, &SyncManager::dataProcessFinished);
colnod->processData(databaseDataToProcess);
return true;
}
void SyncManager::updateLastUpdate(qint64 syncBegin)
{
Timestamp time;
time.set_timestamp(syncBegin);
colnod->getTokensUpdated(time);
database->setLastUpdate(databaseDataToProcess->deletedTokens);
connect(colnod.get(), &ColnodAPI::partOfDataReady, [=](const std::shared_ptr<Data> &data){
database->setLastUpdate(data->updatedTokens);
emit syncFinished();
});
}
void SyncManager::areDataReady()
{
if (++dataAmount == 1) {
qInfo(syncmanager) << "DOWNLOADING DATA -> FINISHED";
qInfo(syncmanager) << equals;
emit dataReady();
}
}
void SyncManager::saveColnodData(const std::shared_ptr<Data> &data)
{
colnodData = data;
areDataReady();
}
void SyncManager::saveDatabaseData(const std::shared_ptr<Data> &data)
{
databaseData = data;
areDataReady();
} }

36
src/colnod/syncmanager.h
View File

@ -8,27 +8,32 @@ Q_DECLARE_LOGGING_CATEGORY(syncmanager)
class SyncManager : public QObject class SyncManager : public QObject
{ {
Q_OBJECT Q_OBJECT
friend class SyncManagerTest;
public: public:
explicit SyncManager(bool dryMode); explicit SyncManager(bool dryMode);
bool databaseInitialization(); bool databaseInitialization();
bool startSynchronization(); bool startSynchronization();
// New approach
private: private:
bool downloadData(); /**
bool prepareData(); * @brief Preapre data to be pushed to database and colnod from actual downloaded data
bool processData(); * @param data from database
* @param data from colnod
* @return prepared data to be pushed into colnod and db
*
* @todo rename when removed old aproach
*/
std::unique_ptr<PreparedData> prepareDataNew(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData);
std::unique_ptr<PreparedData> prepareDataUsindOld(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData);
private:
void initNewToken(std::shared_ptr<Token> token);
void updateLastUpdate(qint64 syncBegin); void updateLastUpdate(qint64 syncBegin);
private:
Q_SLOT void saveColnodData(const std::shared_ptr<Data> &data);
Q_SLOT void saveDatabaseData(const std::shared_ptr<Data> &data);
Q_SLOT void areDataReady();
signals: signals:
void dataReady();
void dataProcessFinished();
void syncFinished(); void syncFinished();
void error(); void error();
@ -40,14 +45,5 @@ private:
serversdk::DatabaseManager dbMan; serversdk::DatabaseManager dbMan;
Timestamp mlastSync; Timestamp mlastSync;
std::shared_ptr<Data> colnodData;
std::shared_ptr<Data> databaseData;
std::shared_ptr<Data> colnodDataToProcess;
std::shared_ptr<Data> databaseDataToProcess;
int dataAmount = 0;
bool mDryMode; bool mDryMode;
std::function<void()> onError = nullptr;
}; };

18
test/auto/colnod/tst_colnodapi.cpp
View File

@ -27,9 +27,6 @@ private:
QString getLoginToken(ColnodAPI &colnod); QString getLoginToken(ColnodAPI &colnod);
}; };
void ColnodTest::login() void ColnodTest::login()
{ {
serversdk::RequestManager reqMan; serversdk::RequestManager reqMan;
@ -52,7 +49,8 @@ void ColnodTest::getTokens()
QSignalSpy spy(login.get(), &serversdk::Request::finished); QSignalSpy spy(login.get(), &serversdk::Request::finished);
spy.wait(); spy.wait();
QString token = getLoginToken(colnod); QString token = getLoginToken(colnod);
Timestamp time(QDateTime::currentMSecsSinceEpoch()); Timestamp time;
time.setTimestamp(QDateTime::currentMSecsSinceEpoch());
colnod.getTokensUpdated(time); colnod.getTokensUpdated(time);
} }
@ -62,7 +60,7 @@ void ColnodTest::download_data_test()
ColnodAPI colnod(&reqMan); ColnodAPI colnod(&reqMan);
{ {
QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished); QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished);
colnod.downloadData(Timestamp(1570190399259)); colnod.downloadData(1570190399259);
QVERIFY(spy.wait()); QVERIFY(spy.wait());
auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0)); auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0));
@ -72,14 +70,13 @@ void ColnodTest::download_data_test()
QVERIFY(data->updatedTokens.contains("5e2f1260-da1c-441f-9644-24dfe3b97b2a")); QVERIFY(data->updatedTokens.contains("5e2f1260-da1c-441f-9644-24dfe3b97b2a"));
} }
{ {
Timestamp time(QDateTime::currentMSecsSinceEpoch()); colnod.downloadData(QDateTime::currentMSecsSinceEpoch());
colnod.downloadData(time);
QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished); QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished);
QVERIFY(spy.wait()); QVERIFY(spy.wait());
auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0)); auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0));
QVERIFY(data); QVERIFY(data);
QVERIFY(data->updatedTokens.size() == 0); QVERIFY(data->updatedTokens.isEmpty());
} }
} }
@ -88,17 +85,14 @@ void ColnodTest::download_data_fail_test()
serversdk::RequestManager reqMan; serversdk::RequestManager reqMan;
ColnodAPI colnod(&reqMan, false, "spaceti", "wrong password"); ColnodAPI colnod(&reqMan, false, "spaceti", "wrong password");
Timestamp time(0);
QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished); QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished);
colnod.downloadData(time); colnod.downloadData(0);
QVERIFY(spy.wait()); QVERIFY(spy.wait());
auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0)); auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0));
QVERIFY(!data); QVERIFY(!data);
} }
QString ColnodTest::getLoginToken(ColnodAPI &colnod) QString ColnodTest::getLoginToken(ColnodAPI &colnod)

15
test/auto/syncmanager/tst_syncmanager.cpp
View File

@ -8,16 +8,29 @@ class SyncManagerTest : public QObject
Q_OBJECT Q_OBJECT
private slots: private slots:
void sync_init_test(); void sync_init_test();
void prepareData_test();
}; };
void SyncManagerTest::sync_init_test() void SyncManagerTest::sync_init_test()
{ {
SyncManager sync(true); SyncManager sync(true);
QVERIFY(sync.databaseInitialization()); QVERIFY(sync.databaseInitialization());
} }
void SyncManagerTest::prepareData_test()
{
SyncManager sync(true);
auto dbData = std::make_shared<Data>();
auto colnodData = std::make_shared<Data>();
auto oldPreparedData = sync.prepareDataUsindOld(dbData, colnodData);
auto newPreparedData = sync.prepareDataNew(dbData, colnodData);
QCOMPARE(newPreparedData->databaseData->updatedTokens.size(), oldPreparedData->databaseData->updatedTokens.size());
QCOMPARE(newPreparedData->databaseData->updatedTokens.size(), oldPreparedData->databaseData->updatedTokens.size());
}
QTEST_MAIN(SyncManagerTest) QTEST_MAIN(SyncManagerTest)
#include "tst_syncmanager.moc" #include "tst_syncmanager.moc"