Sync of changed data in colnod to our db is ready

src/app/main.cpp

@@ -11,7 +11,7 @@ int main(int argc, char *argv[])
     api->downloadData(); // this will start downloading data
     QObject::connect(api, &ColnodAPI::dataDownloaded, [=](){
         qDebug() << "now I have all data";
-        api->syncData();
+        api->syncDataToColnod();
         QCoreApplication::quit();
     });
 

src/colnod/colnodapi.cpp

@@ -4,7 +4,7 @@ ColnodAPI::ColnodAPI()
 {
     reqMan = new RequestManager();
     dataMan = new DataManager();
-    }
+}
 
 
 // PUBLIC FUNCTIONS
@@ -32,9 +32,96 @@ void ColnodAPI::downloadData()
     */
 }
 
-void ColnodAPI::syncData()
+void ColnodAPI::downloadChangedDataFromColnod()
 {
-    dataMan->pushDataToDB();
+    downloadUpdatedData();
+    downloadDeletedData();
+}
+
+void ColnodAPI::downloadUpdatedData()
+{
+    uint64_t lastUpdate = dataMan->getLastUpdate();
+    qDebug() << lastUpdate;
+    Response* resp0 = reqMan->getSubjectsUpdated(lastUpdate);
+    connect(resp0, &Response::ready, [=](){
+        dataMan->localSaveSubjects(resp0->getJson());
+        areChangedDataReady();
+        resp0->deleteLater();
+    });
+
+    Response* resp1 = reqMan->getTokensUpdated(lastUpdate);
+    connect(resp1, &Response::ready, [=](){
+        dataMan->localSaveTokens(resp1->getJson());
+        areChangedDataReady();
+        resp1->deleteLater();
+    });
+}
+
+void ColnodAPI::downloadDeletedData()
+{
+    uint64_t lastUpdate = dataMan->getLastUpdate();
+    Response* resp2 = reqMan->getSubjectsDeleted(lastUpdate);
+    connect(resp2, &Response::ready, [=](){
+        dataMan->localSaveSubjectsToDelete(resp2->getJson());
+        areChangedDataReady();
+        resp2->deleteLater();
+    });
+
+    Response* resp3 = reqMan->getTokensDeleted(lastUpdate);
+    connect(resp3, &Response::ready, [=](){
+        dataMan->localSaveTokensToDelete(resp3->getJson());
+        areChangedDataReady();
+        resp3->deleteLater();
+    });
+
+}
+
+void ColnodAPI::syncDataFromColnod()
+{
+    dataMan->pushTokensToDB();
+    dataMan->pushSubjectsToDB();
+    dataMan->updateLastSync();
+    dataMan->clear();
+}
+
+void ColnodAPI::syncChangedDataFromColnod()
+{
+    syncUpdatedDataFromColnod();
+    syncDeletedDataFromColnod();
+    dataMan->updateLastSync();
+    dataMan->clear();
+}
+
+void ColnodAPI::syncUpdatedDataFromColnod()
+{
+    dataMan->pushSubjectsToDB();
+    dataMan->pushTokensToDB();
+}
+
+
+void ColnodAPI::syncDeletedDataFromColnod()
+{
+    dataMan->deleteSubjectsFromDB();
+    dataMan->deleteTokensFromDB();
+}
+
+
+
+
+void ColnodAPI::syncDataToColnod()
+{
+    // TODO
+}
+
+
+void ColnodAPI::syncChangedDataToColnod()
+{
+    // TODO
+}
+
+void ColnodAPI::prepareDatabase()
+{
+    dataMan->emptyDatabase();
 }
 
 // PRIVATE FUNCTIONS
@@ -44,4 +131,11 @@ void ColnodAPI::areDataReady()
         emit dataDownloaded();
 }
 
+void ColnodAPI::areChangedDataReady()
+{
+    if (dataMan->haveChangedData())
+        emit dataDownloaded();
+}
+
+
 

src/colnod/colnodapi.h

@@ -15,17 +15,35 @@ public:
 
     ColnodAPI();
     void downloadData();
-    void syncData();
+    void syncDataFromColnod();
+    void syncDataToColnod();
+
+
+    void downloadChangedDataFromColnod();
+    void syncChangedDataFromColnod();
+    void syncChangedDataToColnod();
+
+
+
+    void prepareDatabase();
 
 signals:
     void dataDownloaded();
     void partReady();
 
 private slots:
-    void areDataReady();
 
 
 private:
+    void downloadUpdatedData();
+    void downloadDeletedData();
+    void syncUpdatedDataFromColnod();
+    void syncDeletedDataFromColnod();
+    void areDataReady();
+    void areChangedDataReady();
+
+
+
     RequestManager* reqMan = nullptr;
    // Data* data = nullptr;
     DataManager* dataMan = nullptr;

src/colnod/databasemanager.cpp

@@ -8,6 +8,9 @@ DatabaseManager::DatabaseManager(QString host, QString dbName, QString user, QSt
     db.setDatabaseName(dbName);
     db.setHostName(host);
     db.setUserName(user);
+    // home-office stuff
+    //db.setPort(80);
+    //
     db.setPassword(password);
     opened = db.open();
     if (!opened)
@@ -17,25 +20,44 @@ DatabaseManager::DatabaseManager(QString host, QString dbName, QString user, QSt
     query = QSqlQuery("query", db);
 }
 
-bool DatabaseManager::saveSubject(Subject* subject, QString table)
+void DatabaseManager::saveSubject(Subject* subject, QString table)
 {
     QString id = subject->idToStringWithoutBraces();
     if (checkIfExists(id, table))
-        return false;
+    {
+        query.prepare(QString("UPDATE %1 SET name = :name, last_update = :last_update "
+                              "WHERE id = :id)").arg(table));
+        query.bindValue(":id", id);
+        query.bindValue(":name", subject->getName());
+        query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
+        query.exec();
+    }
+    else
+    {
         query.prepare(QString("INSERT INTO %1 (id, name, last_update) "
                               "VALUES (:id, :name, :last_update)").arg(table));
         query.bindValue(":id", id);
         query.bindValue(":name", subject->getName());
         query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
         query.exec();
-    return true;
+    }
 }
 
-bool DatabaseManager::saveToken(Token* token, QString table)
+void DatabaseManager::saveToken(Token* token, QString table)
 {
     QString id = token->idToStringWithoutBraces();
     if (checkIfExists(id, table))
-        return false;
+    {
+        query.prepare(QString("UPDATE %1 SET last_update = :last_update, token_data = :data, subject_id = :subject_id) "
+                      "WHERE id = :id").arg(table));
+        query.bindValue(":id", id);
+        query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(token->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
+        query.bindValue(":data", token->getTokenData());
+        query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces());
+        query.exec();
+    }
+    else
+    {
         query.prepare(QString("INSERT INTO %1 (id, last_update, token_data, subject_id) "
                       "VALUES (:id, :last_update, :data, :subject_id)").arg(table));
         query.bindValue(":id", id);
@@ -43,23 +65,42 @@ bool DatabaseManager::saveToken(Token* token, QString table)
         query.bindValue(":data", token->getTokenData());
         query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces());
         query.exec();
-    return true;
+    }
 }
 
 void DatabaseManager::saveSubjectAttribute(Subject *subject, QString table)
 {
     auto attributes = subject->getAtr();
     for (auto attribute : attributes){
-        query.prepare(QString("INSERT INTO %1 (id, attribute_key, attribute_value, subject_id)"
+        QString id = subject->idToStringWithoutBraces() + " " + attribute.first;
-                              "VALUES (:id, :key, :value, :subject_id)").arg(table));
+        //if (checkIfExists(id, table))
-        query.bindValue(":id", subject->idToStringWithoutBraces() + " " + attribute.first);
+        {
+            query.prepare(QString("UPDATE %1 SET id = :id, attribute_key = :key, attribute_value = :value, subject_id = :subject_id) "
+                                  "WHERE id = :id").arg(table));
+            query.bindValue(":id", id);
             query.bindValue(":key", attribute.first);
             query.bindValue(":value", attribute.second);
             query.bindValue(":subject_id", subject->idToStringWithoutBraces());
             query.exec();
         }
+       /* else
+        {
+            query.prepare(QString("INSERT INTO %1 (id, attribute_key, attribute_value, subject_id)"
+                                  "VALUES (:id, :key, :value, :subject_id)").arg(table));
+            query.bindValue(":id", id);
+            query.bindValue(":key", attribute.first);
+            query.bindValue(":value", attribute.second);
+            query.bindValue(":subject_id", subject->idToStringWithoutBraces());
+            query.exec();
+        }*/
+    }
+}
 
-
+void DatabaseManager::deleteItem(QString id, QString table)
+{
+    query.prepare(QString("DELETE FROM %1 WHERE id = :id").arg(table));
+    query.bindValue(":id", id);
+    query.exec();
 }
 
 QString DatabaseManager::getTestMessage(int id)
@@ -80,7 +121,6 @@ int DatabaseManager::createTestRecord(QString msg)
     query.bindValue(":id", id);
     query.bindValue(":msg", msg);
     query.exec();
-    query.next();
     qDebug() << "U inserted new row, id: " << id << "msg: " << getTestMessage(id);
     return id;
 }
@@ -88,9 +128,61 @@ int DatabaseManager::createTestRecord(QString msg)
 
 bool DatabaseManager::checkIfExists(QString id, QString table)
 {
-    query.prepare(QString("SELECT exists(SELECT 1 FROM %1 WHERE id = :id)").arg(table));
+    query.prepare(QString("SELECT EXISTS(SELECT 1 FROM %1 WHERE id = :id)").arg(table));
     query.bindValue(":id", id);
     query.exec();
-    query.next();
+    query.first();
     return query.value(0).toBool();
 }
+
+void DatabaseManager::dropTables()
+{
+    query.exec("DROP TABLE IF EXISTS colnod_token, colnod_subject, colnod_subject_attribute");
+}
+
+void DatabaseManager::createTables()
+{
+    query.exec("CREATE TABLE colnod_subject ("
+               "id CHAR(36) primary key,"
+               "name VARCHAR(255),"
+               "last_update timestamp)");
+
+    query.exec("CREATE TABLE colnod_token ("
+               "id CHAR(36) primary key,"
+               "token_data VARCHAR(255),"
+               "subject_id CHAR(36),"
+               "last_update timestamp)");
+    query.exec("CREATE TABLE colnod_subject_attribute ("
+               "id VARCHAR(255) primary key,"
+               "attribute_key VARCHAR(255),"
+               "attribute_value VARCHAR(255),"
+               "subject_id CHAR(36))");
+    query.exec("CREATE TABLE users ("
+               "id VARCHAR(255) primary key,"
+               "email VARCHAR(255),"
+               "token_data VARCHAR(255),"
+               "last_update timestamp)");
+}
+
+uint64_t DatabaseManager::getLastUpdate()
+{
+    query.exec("SELECT EXTRACT(EPOCH FROM (SELECT MIN(last_sync) FROM changes))*1000");
+    query.next();
+    return query.value(0).toULongLong();
+}
+
+void DatabaseManager::makeTestingLast_sync()
+{
+    query.prepare("INSERT INTO last_changes (last_sync) "
+               "VALUES (:time)");
+    query.bindValue(":time", QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz"));
+    query.exec();
+}
+
+void DatabaseManager::updateLastSync()
+{
+    query.prepare("INSERT INTO changes (last_sync) "
+               "VALUES (:time)");
+    query.bindValue(":time", QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz"));
+    query.exec();
+}

src/colnod/databasemanager.h

@@ -18,15 +18,23 @@ class DatabaseManager : public QObject
 public:
     DatabaseManager(QString host, QString dbName, QString user, QString password);
 
-    bool saveSubject(Subject* subject, QString table);
+    void saveSubject(Subject* subject, QString table);
-    bool saveToken(Token* token, QString table);
+    void saveToken(Token* token, QString table);
     void saveSubjectAttribute(Subject* subject, QString table);
+    void deleteItem(QString id, QString table);
 
     QString getTestMessage(int id);
     int createTestRecord(QString msg);
+    //
+    void makeTestingLast_sync();
+    //
 
+    void dropTables();
+    void createTables();
 
     bool getOpened() {return opened;}
+    uint64_t getLastUpdate();
+    void updateLastSync();
 
 
 private:

src/colnod/datamanager.cpp

@@ -3,60 +3,191 @@
 DataManager::DataManager()
 {
     db = new DatabaseManager("192.168.1.94", "deloitte", "postgres", "34rjkciea12");
+    //db = new DatabaseManager("localhost", "deloitte", "postgres", "34rjkciea12");
 }
 
 void DataManager::localSaveSubjects(QString response)
 {
     QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
     QJsonArray items = doc.object()["items"].toArray();
+    if (items.size() > 0)
+    {
         for (QJsonValueRef item : items)
         {
             auto subject = new Subject();
             subject->fromJSON(item.toObject());
             subjects.append(subject);
         }
+    }
+    else
+    {
+        Subject* subject = new Subject;
+        subject->setName("empty");
+        subjects.append(subject);
+    }
 }
 
 void DataManager::localSaveTokens(QString response)
 {
     QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
     QJsonArray items = doc.object()["items"].toArray();
+    if (items.size() > 0)
+    {
         for (QJsonValueRef item : items)
         {
             auto token = new Token();
             token->fromJSON(item.toObject());
             tokens.append(token);
-
+        }
+    }
+    else
+    {
+        Token* token = new Token;
+        token->setName("empty");
+        tokens.append(token);
     }
 }
 
-void DataManager::pushDataToDB()
+void DataManager::localSaveSubjectsToDelete(QString json)
 {
-    QSqlDatabase::database().transaction();
+    QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8());
-    qDebug() << "In progress...";
+    QJsonArray items = doc.object()["items"].toArray();
-    int counter = 0;
+    if (items.size() > 0)
+    {
+        for (QJsonValueRef item : items)
+        {
+            subjectsToDelete.append(item.toString());
+        }
+    }
+    else
+    {
+        subjectsToDelete.append("empty");
+    }
+}
+
+void DataManager::localSaveTokensToDelete(QString json)
+{
+    QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8());
+    QJsonArray items = doc.object()["items"].toArray();
+    if (items.size() > 0)
+    {
+        for (QJsonValueRef item : items)
+        {
+            tokensToDelete.append(item.toString());
+        }
+    }
+    else
+    {
+        tokensToDelete.append("empty");
+    }
+}
+
+void DataManager::pushSubjectsToDB()
+{
+    if (subjects.size() == 1 && subjects[0]->getName() == "empty")
+    {
+        qDebug() << "All subjects are up to date";
+        return;
+    }
+    //QSqlDatabase::database().transaction();
+    qDebug() << "Subjects in progress...";
     for (auto subject : subjects)
     {
-        if (db->saveSubject(subject, "subject"))
+        db->saveSubject(subject, "colnod_subject");
-            counter++;
+        db->saveSubjectAttribute(subject, "colnod_subject_attribute");
-        db->saveSubjectAttribute(subject, "subject_attribute");
     }
-    qDebug() << counter << " subjects has been updated";
+    //QSqlDatabase::database().commit();
-    counter = 0;
+    qDebug() << subjects.size() << " subjects has been updated";
-    for (auto token : tokens)
+
-    {
-        if (db->saveToken(token, "token"))
-            counter++;
-    }
-    QSqlDatabase::database().commit();
-    qDebug() << counter << " tokens has been updated";
 }
 
-bool DataManager::haveData() {
+void DataManager::pushTokensToDB()
+{
+    if (tokens.size() == 1 && tokens[0]->getName() == "empty")
+    {
+        qDebug() << "All tokens are up to date";
+        return;
+    }
+    //QSqlDatabase::database().transaction();
+    qDebug() << "Tokens in progress...";
+    for (auto token : tokens)
+    {
+        db->saveToken(token, "colnod_token");
+    }
+    //QSqlDatabase::database().commit();
+    qDebug() << tokens.size() << " tokens has been updated";
+}
+
+void DataManager::deleteSubjectsFromDB()
+{
+    if (subjectsToDelete.size() == 1 && subjectsToDelete[0] == "empty")
+    {
+        qDebug() << "No subjects to be deleted";
+        return;
+    }
+    qDebug() << "Deleted subjects in progress...";
+    //QSqlDatabase::database().transaction();
+    for (auto subjectId : subjectsToDelete)
+    {
+        db->deleteItem(subjectId, "colnod_subject");
+    }
+    //QSqlDatabase::database().commit();
+    qDebug() << subjectsToDelete.size() << "subject has beed deleted";
+}
+
+void DataManager::deleteTokensFromDB()
+{
+    if (tokensToDelete.size() == 1 && tokensToDelete[0] == "empty")
+    {
+        qDebug() << "No tokens to be deleted";
+        return;
+    }
+   // QSqlDatabase::database().transaction();
+    qDebug() << "Deleted tokens in progress...";
+    for (auto tokenId : tokensToDelete)
+    {
+        db->deleteItem(tokenId, "colnod_token");
+    }
+    //QSqlDatabase::database().commit();
+    qDebug() << tokensToDelete.size() << "tokens has beed deleted";
+}
+
+bool DataManager::haveData()
+{
   if ( !subjects.isEmpty() && !tokens.isEmpty() )
     return true;
-
   return false;
 }
 
+bool DataManager::haveChangedData()
+{
+    if ( !subjects.isEmpty() && !tokens.isEmpty() && !subjectsToDelete.isEmpty() && !tokensToDelete.isEmpty() )
+        return true;
+    return false;
+}
+
+uint64_t DataManager::getLastUpdate()
+{
+    return db->getLastUpdate();
+}
+
+void DataManager::emptyDatabase()
+{
+    db->dropTables();
+    db->createTables();
+}
+
+void DataManager::clear()
+{
+    subjects.clear();
+    subjectsToDelete.clear();
+    tokens.clear();
+    tokensToDelete.clear();
+}
+
+void DataManager::updateLastSync()
+{
+    db->updateLastSync();
+}
+
 

src/colnod/datamanager.h

@@ -19,19 +19,29 @@ public:
 
     void localSaveSubjects(QString jsonResponse);
     void localSaveTokens(QString jsonResponse);
-    void pushDataToDB();
+    void localSaveSubjectsToDelete(QString jsonResponse);
+    void localSaveTokensToDelete(QString jsonResponse);
+
+    void pushSubjectsToDB();
+    void pushTokensToDB();
+    void deleteSubjectsFromDB();
+    void deleteTokensFromDB();
+
+    void updateLastSync();
+    uint64_t getLastUpdate();
+
     bool haveData();
+    bool haveChangedData();
 
-
+    void emptyDatabase();
-
+    void clear();
 
 private:
     DatabaseManager *db = nullptr;
     QList<QPointer<Subject>> subjects;
     QList<QPointer<Token>> tokens;
-    //Data *dataToProcess = nullptr;
+    QList<QString> subjectsToDelete;
-
+    QList<QString> tokensToDelete;
-
 };
 
 #endif // DATAMANAGER_H

src/colnod/requestmanager.h

@@ -84,7 +84,7 @@ private:
     QJsonDocument config;
     QString authHeader;
     QString host = "https://192.168.1.3:8443";
-    //QString host = "http://localhost:8080";
+   // QString host = "https://localhost:8080";
 
     QString password = "spaceti1";
     QString user = "spaceti";

test/unitTests/UnitTest.cpp

@@ -46,8 +46,8 @@ void UnitTest::getSubjects()
 void UnitTest::getTokens()
 {
     RequestManager* testAccess = new RequestManager();
-    QSignalSpy spy(testAccess, SIGNAL(responseReady()));
     Response* response = testAccess->getTokens();
+    QSignalSpy spy(response, &Response::ready);
     QCOMPARE(spy.count(), 0);
     spy.wait(15000);
     QVERIFY(spy.isValid());

test/unitTests/complextest.cpp

@@ -4,6 +4,7 @@ void ComplexTest::initTestCase()
 {
     reqMan = new RequestManager;
     start = QDateTime::currentMSecsSinceEpoch()-2000;
+    api = new ColnodAPI;
     qDebug() << start;
 }
 
@@ -99,6 +100,33 @@ void ComplexTest::complexTest()
 }
 
 
+void ComplexTest::fullSync()
+{
+    api->prepareDatabase();
+    api->downloadData();
+    QSignalSpy spy(api, &ColnodAPI::dataDownloaded);
+    spy.wait(10000);
+    api->syncDataFromColnod();
+   // api->syncDataToColnod();
+}
+
+void ComplexTest::syncLastUpdated()
+{
+    api->downloadChangedDataFromColnod();
+    QSignalSpy spy(api, &ColnodAPI::dataDownloaded);
+    spy.wait(10000);
+    api->syncChangedDataFromColnod();
+    //api->syncUpdatedDataToColnod();
+
+}
+
+void ComplexTest::testingStuff()
+{
+    //DatabaseManager* db = new DatabaseManager("192.168.1.94", "deloitte", "postgres", "34rjkciea12");
+    //DatabaseManager* db = new DatabaseManager("localhost", "deloitte", "postgres", "34rjkciea12");
+    //db->makeTestingLast_sync();
+}
+
 // HELPFUL_FUNCTIONS
 int ComplexTest::getNumOfSubjects()
 {

test/unitTests/complextest.h

@@ -5,6 +5,7 @@
 #include <QtTest/QtTest>
 #include "response.h"
 #include "requestmanager.h"
+#include "colnodapi.h"
 
 class ComplexTest : public QObject
 {
@@ -13,6 +14,9 @@ class ComplexTest : public QObject
 private slots:
     void initTestCase();
     void complexTest();
+    void fullSync();
+    void syncLastUpdated();
+    void testingStuff();
     void cleanupTestCase();
 
 private:
@@ -29,6 +33,6 @@ private:
     QString password = "spaceti1";
     RequestManager* reqMan = nullptr;
     uint64_t start = 0;
-
+    ColnodAPI* api = nullptr;
 };