From ecdb181ca94a0088a2c424cdcc45ce08a397c47c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Fischer?= Date: Thu, 17 Oct 2019 10:46:54 +0200 Subject: [PATCH] Sync of changed data in colnod to our db is ready --- src/app/main.cpp | 2 +- src/colnod/colnodapi.cpp | 100 +++++++++++++++++- src/colnod/colnodapi.h | 22 +++- src/colnod/databasemanager.cpp | 152 +++++++++++++++++++++------ src/colnod/databasemanager.h | 12 ++- src/colnod/datamanager.cpp | 181 ++++++++++++++++++++++++++++----- src/colnod/datamanager.h | 22 ++-- src/colnod/requestmanager.h | 2 +- test/unitTests/UnitTest.cpp | 2 +- test/unitTests/complextest.cpp | 28 +++++ test/unitTests/complextest.h | 6 +- 11 files changed, 457 insertions(+), 72 deletions(-) diff --git a/src/app/main.cpp b/src/app/main.cpp index 82a5be8..70c5e9e 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -11,7 +11,7 @@ int main(int argc, char *argv[]) api->downloadData(); // this will start downloading data QObject::connect(api, &ColnodAPI::dataDownloaded, [=](){ qDebug() << "now I have all data"; - api->syncData(); + api->syncDataToColnod(); QCoreApplication::quit(); }); diff --git a/src/colnod/colnodapi.cpp b/src/colnod/colnodapi.cpp index ae34e39..504bb14 100644 --- a/src/colnod/colnodapi.cpp +++ b/src/colnod/colnodapi.cpp @@ -4,7 +4,7 @@ ColnodAPI::ColnodAPI() { reqMan = new RequestManager(); dataMan = new DataManager(); - } +} // PUBLIC FUNCTIONS @@ -32,9 +32,96 @@ void ColnodAPI::downloadData() */ } -void ColnodAPI::syncData() +void ColnodAPI::downloadChangedDataFromColnod() { - dataMan->pushDataToDB(); + downloadUpdatedData(); + downloadDeletedData(); +} + +void ColnodAPI::downloadUpdatedData() +{ + uint64_t lastUpdate = dataMan->getLastUpdate(); + qDebug() << lastUpdate; + Response* resp0 = reqMan->getSubjectsUpdated(lastUpdate); + connect(resp0, &Response::ready, [=](){ + dataMan->localSaveSubjects(resp0->getJson()); + areChangedDataReady(); + resp0->deleteLater(); + }); + + Response* resp1 = reqMan->getTokensUpdated(lastUpdate); + connect(resp1, &Response::ready, [=](){ + dataMan->localSaveTokens(resp1->getJson()); + areChangedDataReady(); + resp1->deleteLater(); + }); +} + +void ColnodAPI::downloadDeletedData() +{ + uint64_t lastUpdate = dataMan->getLastUpdate(); + Response* resp2 = reqMan->getSubjectsDeleted(lastUpdate); + connect(resp2, &Response::ready, [=](){ + dataMan->localSaveSubjectsToDelete(resp2->getJson()); + areChangedDataReady(); + resp2->deleteLater(); + }); + + Response* resp3 = reqMan->getTokensDeleted(lastUpdate); + connect(resp3, &Response::ready, [=](){ + dataMan->localSaveTokensToDelete(resp3->getJson()); + areChangedDataReady(); + resp3->deleteLater(); + }); + +} + +void ColnodAPI::syncDataFromColnod() +{ + dataMan->pushTokensToDB(); + dataMan->pushSubjectsToDB(); + dataMan->updateLastSync(); + dataMan->clear(); +} + +void ColnodAPI::syncChangedDataFromColnod() +{ + syncUpdatedDataFromColnod(); + syncDeletedDataFromColnod(); + dataMan->updateLastSync(); + dataMan->clear(); +} + +void ColnodAPI::syncUpdatedDataFromColnod() +{ + dataMan->pushSubjectsToDB(); + dataMan->pushTokensToDB(); +} + + +void ColnodAPI::syncDeletedDataFromColnod() +{ + dataMan->deleteSubjectsFromDB(); + dataMan->deleteTokensFromDB(); +} + + + + +void ColnodAPI::syncDataToColnod() +{ + // TODO +} + + +void ColnodAPI::syncChangedDataToColnod() +{ + // TODO +} + +void ColnodAPI::prepareDatabase() +{ + dataMan->emptyDatabase(); } // PRIVATE FUNCTIONS @@ -44,4 +131,11 @@ void ColnodAPI::areDataReady() emit dataDownloaded(); } +void ColnodAPI::areChangedDataReady() +{ + if (dataMan->haveChangedData()) + emit dataDownloaded(); +} + + diff --git a/src/colnod/colnodapi.h b/src/colnod/colnodapi.h index 22ad551..dd5e3e0 100644 --- a/src/colnod/colnodapi.h +++ b/src/colnod/colnodapi.h @@ -15,17 +15,35 @@ public: ColnodAPI(); void downloadData(); - void syncData(); + void syncDataFromColnod(); + void syncDataToColnod(); + + + void downloadChangedDataFromColnod(); + void syncChangedDataFromColnod(); + void syncChangedDataToColnod(); + + + + void prepareDatabase(); signals: void dataDownloaded(); void partReady(); private slots: - void areDataReady(); private: + void downloadUpdatedData(); + void downloadDeletedData(); + void syncUpdatedDataFromColnod(); + void syncDeletedDataFromColnod(); + void areDataReady(); + void areChangedDataReady(); + + + RequestManager* reqMan = nullptr; // Data* data = nullptr; DataManager* dataMan = nullptr; diff --git a/src/colnod/databasemanager.cpp b/src/colnod/databasemanager.cpp index 75d9d8a..9886500 100644 --- a/src/colnod/databasemanager.cpp +++ b/src/colnod/databasemanager.cpp @@ -8,6 +8,9 @@ DatabaseManager::DatabaseManager(QString host, QString dbName, QString user, QSt db.setDatabaseName(dbName); db.setHostName(host); db.setUserName(user); + // home-office stuff + //db.setPort(80); + // db.setPassword(password); opened = db.open(); if (!opened) @@ -17,49 +20,87 @@ DatabaseManager::DatabaseManager(QString host, QString dbName, QString user, QSt query = QSqlQuery("query", db); } -bool DatabaseManager::saveSubject(Subject* subject, QString table) +void DatabaseManager::saveSubject(Subject* subject, QString table) { QString id = subject->idToStringWithoutBraces(); if (checkIfExists(id, table)) - return false; - query.prepare(QString("INSERT INTO %1 (id, name, last_update) " - "VALUES (:id, :name, :last_update)").arg(table)); - query.bindValue(":id", id); - query.bindValue(":name", subject->getName()); - query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz")); - query.exec(); - return true; + { + query.prepare(QString("UPDATE %1 SET name = :name, last_update = :last_update " + "WHERE id = :id)").arg(table)); + query.bindValue(":id", id); + query.bindValue(":name", subject->getName()); + query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz")); + query.exec(); + } + else + { + query.prepare(QString("INSERT INTO %1 (id, name, last_update) " + "VALUES (:id, :name, :last_update)").arg(table)); + query.bindValue(":id", id); + query.bindValue(":name", subject->getName()); + query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz")); + query.exec(); + } } -bool DatabaseManager::saveToken(Token* token, QString table) +void DatabaseManager::saveToken(Token* token, QString table) { QString id = token->idToStringWithoutBraces(); if (checkIfExists(id, table)) - return false; - query.prepare(QString("INSERT INTO %1 (id, last_update, token_data, subject_id) " - "VALUES (:id, :last_update, :data, :subject_id)").arg(table)); - query.bindValue(":id", id); - query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(token->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz")); - query.bindValue(":data", token->getTokenData()); - query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces()); - query.exec(); - return true; + { + query.prepare(QString("UPDATE %1 SET last_update = :last_update, token_data = :data, subject_id = :subject_id) " + "WHERE id = :id").arg(table)); + query.bindValue(":id", id); + query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(token->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz")); + query.bindValue(":data", token->getTokenData()); + query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces()); + query.exec(); + } + else + { + query.prepare(QString("INSERT INTO %1 (id, last_update, token_data, subject_id) " + "VALUES (:id, :last_update, :data, :subject_id)").arg(table)); + query.bindValue(":id", id); + query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(token->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz")); + query.bindValue(":data", token->getTokenData()); + query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces()); + query.exec(); + } } void DatabaseManager::saveSubjectAttribute(Subject *subject, QString table) { auto attributes = subject->getAtr(); for (auto attribute : attributes){ - query.prepare(QString("INSERT INTO %1 (id, attribute_key, attribute_value, subject_id)" - "VALUES (:id, :key, :value, :subject_id)").arg(table)); - query.bindValue(":id", subject->idToStringWithoutBraces() + " " + attribute.first); - query.bindValue(":key", attribute.first); - query.bindValue(":value", attribute.second); - query.bindValue(":subject_id", subject->idToStringWithoutBraces()); - query.exec(); + QString id = subject->idToStringWithoutBraces() + " " + attribute.first; + //if (checkIfExists(id, table)) + { + query.prepare(QString("UPDATE %1 SET id = :id, attribute_key = :key, attribute_value = :value, subject_id = :subject_id) " + "WHERE id = :id").arg(table)); + query.bindValue(":id", id); + query.bindValue(":key", attribute.first); + query.bindValue(":value", attribute.second); + query.bindValue(":subject_id", subject->idToStringWithoutBraces()); + query.exec(); + } + /* else + { + query.prepare(QString("INSERT INTO %1 (id, attribute_key, attribute_value, subject_id)" + "VALUES (:id, :key, :value, :subject_id)").arg(table)); + query.bindValue(":id", id); + query.bindValue(":key", attribute.first); + query.bindValue(":value", attribute.second); + query.bindValue(":subject_id", subject->idToStringWithoutBraces()); + query.exec(); + }*/ } +} - +void DatabaseManager::deleteItem(QString id, QString table) +{ + query.prepare(QString("DELETE FROM %1 WHERE id = :id").arg(table)); + query.bindValue(":id", id); + query.exec(); } QString DatabaseManager::getTestMessage(int id) @@ -80,7 +121,6 @@ int DatabaseManager::createTestRecord(QString msg) query.bindValue(":id", id); query.bindValue(":msg", msg); query.exec(); - query.next(); qDebug() << "U inserted new row, id: " << id << "msg: " << getTestMessage(id); return id; } @@ -88,9 +128,61 @@ int DatabaseManager::createTestRecord(QString msg) bool DatabaseManager::checkIfExists(QString id, QString table) { - query.prepare(QString("SELECT exists(SELECT 1 FROM %1 WHERE id = :id)").arg(table)); + query.prepare(QString("SELECT EXISTS(SELECT 1 FROM %1 WHERE id = :id)").arg(table)); query.bindValue(":id", id); query.exec(); - query.next(); + query.first(); return query.value(0).toBool(); } + +void DatabaseManager::dropTables() +{ + query.exec("DROP TABLE IF EXISTS colnod_token, colnod_subject, colnod_subject_attribute"); +} + +void DatabaseManager::createTables() +{ + query.exec("CREATE TABLE colnod_subject (" + "id CHAR(36) primary key," + "name VARCHAR(255)," + "last_update timestamp)"); + + query.exec("CREATE TABLE colnod_token (" + "id CHAR(36) primary key," + "token_data VARCHAR(255)," + "subject_id CHAR(36)," + "last_update timestamp)"); + query.exec("CREATE TABLE colnod_subject_attribute (" + "id VARCHAR(255) primary key," + "attribute_key VARCHAR(255)," + "attribute_value VARCHAR(255)," + "subject_id CHAR(36))"); + query.exec("CREATE TABLE users (" + "id VARCHAR(255) primary key," + "email VARCHAR(255)," + "token_data VARCHAR(255)," + "last_update timestamp)"); +} + +uint64_t DatabaseManager::getLastUpdate() +{ + query.exec("SELECT EXTRACT(EPOCH FROM (SELECT MIN(last_sync) FROM changes))*1000"); + query.next(); + return query.value(0).toULongLong(); +} + +void DatabaseManager::makeTestingLast_sync() +{ + query.prepare("INSERT INTO last_changes (last_sync) " + "VALUES (:time)"); + query.bindValue(":time", QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz")); + query.exec(); +} + +void DatabaseManager::updateLastSync() +{ + query.prepare("INSERT INTO changes (last_sync) " + "VALUES (:time)"); + query.bindValue(":time", QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz")); + query.exec(); +} diff --git a/src/colnod/databasemanager.h b/src/colnod/databasemanager.h index 8cddc82..8eb3306 100644 --- a/src/colnod/databasemanager.h +++ b/src/colnod/databasemanager.h @@ -18,15 +18,23 @@ class DatabaseManager : public QObject public: DatabaseManager(QString host, QString dbName, QString user, QString password); - bool saveSubject(Subject* subject, QString table); - bool saveToken(Token* token, QString table); + void saveSubject(Subject* subject, QString table); + void saveToken(Token* token, QString table); void saveSubjectAttribute(Subject* subject, QString table); + void deleteItem(QString id, QString table); QString getTestMessage(int id); int createTestRecord(QString msg); + // + void makeTestingLast_sync(); + // + void dropTables(); + void createTables(); bool getOpened() {return opened;} + uint64_t getLastUpdate(); + void updateLastSync(); private: diff --git a/src/colnod/datamanager.cpp b/src/colnod/datamanager.cpp index 3d06661..8ad85dc 100644 --- a/src/colnod/datamanager.cpp +++ b/src/colnod/datamanager.cpp @@ -3,16 +3,26 @@ DataManager::DataManager() { db = new DatabaseManager("192.168.1.94", "deloitte", "postgres", "34rjkciea12"); + //db = new DatabaseManager("localhost", "deloitte", "postgres", "34rjkciea12"); } void DataManager::localSaveSubjects(QString response) { QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8()); QJsonArray items = doc.object()["items"].toArray(); - for (QJsonValueRef item : items) + if (items.size() > 0) { - auto subject = new Subject(); - subject->fromJSON(item.toObject()); + for (QJsonValueRef item : items) + { + auto subject = new Subject(); + subject->fromJSON(item.toObject()); + subjects.append(subject); + } + } + else + { + Subject* subject = new Subject; + subject->setName("empty"); subjects.append(subject); } } @@ -21,42 +31,163 @@ void DataManager::localSaveTokens(QString response) { QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8()); QJsonArray items = doc.object()["items"].toArray(); - for (QJsonValueRef item : items) + if (items.size() > 0) { - auto token = new Token(); - token->fromJSON(item.toObject()); + for (QJsonValueRef item : items) + { + auto token = new Token(); + token->fromJSON(item.toObject()); + tokens.append(token); + } + } + else + { + Token* token = new Token; + token->setName("empty"); tokens.append(token); - } } -void DataManager::pushDataToDB() +void DataManager::localSaveSubjectsToDelete(QString json) { - QSqlDatabase::database().transaction(); - qDebug() << "In progress..."; - int counter = 0; + QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8()); + QJsonArray items = doc.object()["items"].toArray(); + if (items.size() > 0) + { + for (QJsonValueRef item : items) + { + subjectsToDelete.append(item.toString()); + } + } + else + { + subjectsToDelete.append("empty"); + } +} + +void DataManager::localSaveTokensToDelete(QString json) +{ + QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8()); + QJsonArray items = doc.object()["items"].toArray(); + if (items.size() > 0) + { + for (QJsonValueRef item : items) + { + tokensToDelete.append(item.toString()); + } + } + else + { + tokensToDelete.append("empty"); + } +} + +void DataManager::pushSubjectsToDB() +{ + if (subjects.size() == 1 && subjects[0]->getName() == "empty") + { + qDebug() << "All subjects are up to date"; + return; + } + //QSqlDatabase::database().transaction(); + qDebug() << "Subjects in progress..."; for (auto subject : subjects) { - if (db->saveSubject(subject, "subject")) - counter++; - db->saveSubjectAttribute(subject, "subject_attribute"); + db->saveSubject(subject, "colnod_subject"); + db->saveSubjectAttribute(subject, "colnod_subject_attribute"); } - qDebug() << counter << " subjects has been updated"; - counter = 0; - for (auto token : tokens) - { - if (db->saveToken(token, "token")) - counter++; - } - QSqlDatabase::database().commit(); - qDebug() << counter << " tokens has been updated"; + //QSqlDatabase::database().commit(); + qDebug() << subjects.size() << " subjects has been updated"; + } -bool DataManager::haveData() { +void DataManager::pushTokensToDB() +{ + if (tokens.size() == 1 && tokens[0]->getName() == "empty") + { + qDebug() << "All tokens are up to date"; + return; + } + //QSqlDatabase::database().transaction(); + qDebug() << "Tokens in progress..."; + for (auto token : tokens) + { + db->saveToken(token, "colnod_token"); + } + //QSqlDatabase::database().commit(); + qDebug() << tokens.size() << " tokens has been updated"; +} + +void DataManager::deleteSubjectsFromDB() +{ + if (subjectsToDelete.size() == 1 && subjectsToDelete[0] == "empty") + { + qDebug() << "No subjects to be deleted"; + return; + } + qDebug() << "Deleted subjects in progress..."; + //QSqlDatabase::database().transaction(); + for (auto subjectId : subjectsToDelete) + { + db->deleteItem(subjectId, "colnod_subject"); + } + //QSqlDatabase::database().commit(); + qDebug() << subjectsToDelete.size() << "subject has beed deleted"; +} + +void DataManager::deleteTokensFromDB() +{ + if (tokensToDelete.size() == 1 && tokensToDelete[0] == "empty") + { + qDebug() << "No tokens to be deleted"; + return; + } + // QSqlDatabase::database().transaction(); + qDebug() << "Deleted tokens in progress..."; + for (auto tokenId : tokensToDelete) + { + db->deleteItem(tokenId, "colnod_token"); + } + //QSqlDatabase::database().commit(); + qDebug() << tokensToDelete.size() << "tokens has beed deleted"; +} + +bool DataManager::haveData() +{ if ( !subjects.isEmpty() && !tokens.isEmpty() ) return true; - return false; } +bool DataManager::haveChangedData() +{ + if ( !subjects.isEmpty() && !tokens.isEmpty() && !subjectsToDelete.isEmpty() && !tokensToDelete.isEmpty() ) + return true; + return false; +} + +uint64_t DataManager::getLastUpdate() +{ + return db->getLastUpdate(); +} + +void DataManager::emptyDatabase() +{ + db->dropTables(); + db->createTables(); +} + +void DataManager::clear() +{ + subjects.clear(); + subjectsToDelete.clear(); + tokens.clear(); + tokensToDelete.clear(); +} + +void DataManager::updateLastSync() +{ + db->updateLastSync(); +} + diff --git a/src/colnod/datamanager.h b/src/colnod/datamanager.h index b8bc0d9..c68a128 100644 --- a/src/colnod/datamanager.h +++ b/src/colnod/datamanager.h @@ -19,19 +19,29 @@ public: void localSaveSubjects(QString jsonResponse); void localSaveTokens(QString jsonResponse); - void pushDataToDB(); + void localSaveSubjectsToDelete(QString jsonResponse); + void localSaveTokensToDelete(QString jsonResponse); + + void pushSubjectsToDB(); + void pushTokensToDB(); + void deleteSubjectsFromDB(); + void deleteTokensFromDB(); + + void updateLastSync(); + uint64_t getLastUpdate(); + bool haveData(); + bool haveChangedData(); - - + void emptyDatabase(); + void clear(); private: DatabaseManager *db = nullptr; QList> subjects; QList> tokens; - //Data *dataToProcess = nullptr; - - + QList subjectsToDelete; + QList tokensToDelete; }; #endif // DATAMANAGER_H diff --git a/src/colnod/requestmanager.h b/src/colnod/requestmanager.h index 55dc5be..028ec98 100644 --- a/src/colnod/requestmanager.h +++ b/src/colnod/requestmanager.h @@ -84,7 +84,7 @@ private: QJsonDocument config; QString authHeader; QString host = "https://192.168.1.3:8443"; - //QString host = "http://localhost:8080"; + // QString host = "https://localhost:8080"; QString password = "spaceti1"; QString user = "spaceti"; diff --git a/test/unitTests/UnitTest.cpp b/test/unitTests/UnitTest.cpp index efb0fa2..6da6094 100644 --- a/test/unitTests/UnitTest.cpp +++ b/test/unitTests/UnitTest.cpp @@ -46,8 +46,8 @@ void UnitTest::getSubjects() void UnitTest::getTokens() { RequestManager* testAccess = new RequestManager(); - QSignalSpy spy(testAccess, SIGNAL(responseReady())); Response* response = testAccess->getTokens(); + QSignalSpy spy(response, &Response::ready); QCOMPARE(spy.count(), 0); spy.wait(15000); QVERIFY(spy.isValid()); diff --git a/test/unitTests/complextest.cpp b/test/unitTests/complextest.cpp index 881913e..0d80db9 100644 --- a/test/unitTests/complextest.cpp +++ b/test/unitTests/complextest.cpp @@ -4,6 +4,7 @@ void ComplexTest::initTestCase() { reqMan = new RequestManager; start = QDateTime::currentMSecsSinceEpoch()-2000; + api = new ColnodAPI; qDebug() << start; } @@ -99,6 +100,33 @@ void ComplexTest::complexTest() } +void ComplexTest::fullSync() +{ + api->prepareDatabase(); + api->downloadData(); + QSignalSpy spy(api, &ColnodAPI::dataDownloaded); + spy.wait(10000); + api->syncDataFromColnod(); + // api->syncDataToColnod(); +} + +void ComplexTest::syncLastUpdated() +{ + api->downloadChangedDataFromColnod(); + QSignalSpy spy(api, &ColnodAPI::dataDownloaded); + spy.wait(10000); + api->syncChangedDataFromColnod(); + //api->syncUpdatedDataToColnod(); + +} + +void ComplexTest::testingStuff() +{ + //DatabaseManager* db = new DatabaseManager("192.168.1.94", "deloitte", "postgres", "34rjkciea12"); + //DatabaseManager* db = new DatabaseManager("localhost", "deloitte", "postgres", "34rjkciea12"); + //db->makeTestingLast_sync(); +} + // HELPFUL_FUNCTIONS int ComplexTest::getNumOfSubjects() { diff --git a/test/unitTests/complextest.h b/test/unitTests/complextest.h index 117691b..ed34fe5 100644 --- a/test/unitTests/complextest.h +++ b/test/unitTests/complextest.h @@ -5,6 +5,7 @@ #include #include "response.h" #include "requestmanager.h" +#include "colnodapi.h" class ComplexTest : public QObject { @@ -13,6 +14,9 @@ class ComplexTest : public QObject private slots: void initTestCase(); void complexTest(); + void fullSync(); + void syncLastUpdated(); + void testingStuff(); void cleanupTestCase(); private: @@ -29,6 +33,6 @@ private: QString password = "spaceti1"; RequestManager* reqMan = nullptr; uint64_t start = 0; - + ColnodAPI* api = nullptr; };