michalks/colnod-connector
1
0
Fork 0
forked from ondra/colnod-connector
Code Issues Pull Requests Projects Releases Wiki Activity

Sync of changed data in colnod to our db is ready

Browse Source
This commit is contained in:
Ondřej Fischer 2019-10-17 10:46:54 +02:00
parent 796b8a7521
commit ecdb181ca9
11 changed files with 457 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/app/main.cpp
View File

@ -11,7 +11,7 @@ int main(int argc, char *argv[])
api->downloadData(); // this will start downloading data
QObject::connect(api, &ColnodAPI::dataDownloaded, [=](){
qDebug() << "now I have all data";
api->syncData();
api->syncDataToColnod();
QCoreApplication::quit();
});

100
src/colnod/colnodapi.cpp
View File

@ -4,7 +4,7 @@ ColnodAPI::ColnodAPI()
{
reqMan = new RequestManager();
dataMan = new DataManager();
}
}
// PUBLIC FUNCTIONS
@ -32,9 +32,96 @@ void ColnodAPI::downloadData()
*/
}
void ColnodAPI::syncData()
void ColnodAPI::downloadChangedDataFromColnod()
{
dataMan->pushDataToDB();
downloadUpdatedData();
downloadDeletedData();
}
void ColnodAPI::downloadUpdatedData()
{
uint64_t lastUpdate = dataMan->getLastUpdate();
qDebug() << lastUpdate;
Response* resp0 = reqMan->getSubjectsUpdated(lastUpdate);
connect(resp0, &Response::ready, [=](){
dataMan->localSaveSubjects(resp0->getJson());
areChangedDataReady();
resp0->deleteLater();
});
Response* resp1 = reqMan->getTokensUpdated(lastUpdate);
connect(resp1, &Response::ready, [=](){
dataMan->localSaveTokens(resp1->getJson());
areChangedDataReady();
resp1->deleteLater();
});
}
void ColnodAPI::downloadDeletedData()
{
uint64_t lastUpdate = dataMan->getLastUpdate();
Response* resp2 = reqMan->getSubjectsDeleted(lastUpdate);
connect(resp2, &Response::ready, [=](){
dataMan->localSaveSubjectsToDelete(resp2->getJson());
areChangedDataReady();
resp2->deleteLater();
});
Response* resp3 = reqMan->getTokensDeleted(lastUpdate);
connect(resp3, &Response::ready, [=](){
dataMan->localSaveTokensToDelete(resp3->getJson());
areChangedDataReady();
resp3->deleteLater();
});
}
void ColnodAPI::syncDataFromColnod()
{
dataMan->pushTokensToDB();
dataMan->pushSubjectsToDB();
dataMan->updateLastSync();
dataMan->clear();
}
void ColnodAPI::syncChangedDataFromColnod()
{
syncUpdatedDataFromColnod();
syncDeletedDataFromColnod();
dataMan->updateLastSync();
dataMan->clear();
}
void ColnodAPI::syncUpdatedDataFromColnod()
{
dataMan->pushSubjectsToDB();
dataMan->pushTokensToDB();
}
void ColnodAPI::syncDeletedDataFromColnod()
{
dataMan->deleteSubjectsFromDB();
dataMan->deleteTokensFromDB();
}
void ColnodAPI::syncDataToColnod()
{
// TODO
}
void ColnodAPI::syncChangedDataToColnod()
{
// TODO
}
void ColnodAPI::prepareDatabase()
{
dataMan->emptyDatabase();
}
// PRIVATE FUNCTIONS
@ -44,4 +131,11 @@ void ColnodAPI::areDataReady()
emit dataDownloaded();
}
void ColnodAPI::areChangedDataReady()
{
if (dataMan->haveChangedData())
emit dataDownloaded();
}

22
src/colnod/colnodapi.h
View File

@ -15,17 +15,35 @@ public:
ColnodAPI();
void downloadData();
void syncData();
void syncDataFromColnod();
void syncDataToColnod();
void downloadChangedDataFromColnod();
void syncChangedDataFromColnod();
void syncChangedDataToColnod();
void prepareDatabase();
signals:
void dataDownloaded();
void partReady();
private slots:
void areDataReady();
private:
void downloadUpdatedData();
void downloadDeletedData();
void syncUpdatedDataFromColnod();
void syncDeletedDataFromColnod();
void areDataReady();
void areChangedDataReady();
RequestManager* reqMan = nullptr;
// Data* data = nullptr;
DataManager* dataMan = nullptr;

152
src/colnod/databasemanager.cpp
View File

@ -8,6 +8,9 @@ DatabaseManager::DatabaseManager(QString host, QString dbName, QString user, QSt
db.setDatabaseName(dbName);
db.setHostName(host);
db.setUserName(user);
// home-office stuff
//db.setPort(80);
//
db.setPassword(password);
opened = db.open();
if (!opened)
@ -17,49 +20,87 @@ DatabaseManager::DatabaseManager(QString host, QString dbName, QString user, QSt
query = QSqlQuery("query", db);
}
bool DatabaseManager::saveSubject(Subject* subject, QString table)
void DatabaseManager::saveSubject(Subject* subject, QString table)
{
QString id = subject->idToStringWithoutBraces();
if (checkIfExists(id, table))
return false;
query.prepare(QString("INSERT INTO %1 (id, name, last_update) "
"VALUES (:id, :name, :last_update)").arg(table));
query.bindValue(":id", id);
query.bindValue(":name", subject->getName());
query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.exec();
return true;
{
query.prepare(QString("UPDATE %1 SET name = :name, last_update = :last_update "
"WHERE id = :id)").arg(table));
query.bindValue(":id", id);
query.bindValue(":name", subject->getName());
query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.exec();
}
else
{
query.prepare(QString("INSERT INTO %1 (id, name, last_update) "
"VALUES (:id, :name, :last_update)").arg(table));
query.bindValue(":id", id);
query.bindValue(":name", subject->getName());
query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(subject->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.exec();
}
}
bool DatabaseManager::saveToken(Token* token, QString table)
void DatabaseManager::saveToken(Token* token, QString table)
{
QString id = token->idToStringWithoutBraces();
if (checkIfExists(id, table))
return false;
query.prepare(QString("INSERT INTO %1 (id, last_update, token_data, subject_id) "
"VALUES (:id, :last_update, :data, :subject_id)").arg(table));
query.bindValue(":id", id);
query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(token->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.bindValue(":data", token->getTokenData());
query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces());
query.exec();
return true;
{
query.prepare(QString("UPDATE %1 SET last_update = :last_update, token_data = :data, subject_id = :subject_id) "
"WHERE id = :id").arg(table));
query.bindValue(":id", id);
query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(token->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.bindValue(":data", token->getTokenData());
query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces());
query.exec();
}
else
{
query.prepare(QString("INSERT INTO %1 (id, last_update, token_data, subject_id) "
"VALUES (:id, :last_update, :data, :subject_id)").arg(table));
query.bindValue(":id", id);
query.bindValue(":last_update", QDateTime::fromMSecsSinceEpoch(token->getLastUpdate()).toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.bindValue(":data", token->getTokenData());
query.bindValue(":subject_id", token->subjectIdToStringWithoutBraces());
query.exec();
}
}
void DatabaseManager::saveSubjectAttribute(Subject *subject, QString table)
{
auto attributes = subject->getAtr();
for (auto attribute : attributes){
query.prepare(QString("INSERT INTO %1 (id, attribute_key, attribute_value, subject_id)"
"VALUES (:id, :key, :value, :subject_id)").arg(table));
query.bindValue(":id", subject->idToStringWithoutBraces() + " " + attribute.first);
query.bindValue(":key", attribute.first);
query.bindValue(":value", attribute.second);
query.bindValue(":subject_id", subject->idToStringWithoutBraces());
query.exec();
QString id = subject->idToStringWithoutBraces() + " " + attribute.first;
//if (checkIfExists(id, table))
{
query.prepare(QString("UPDATE %1 SET id = :id, attribute_key = :key, attribute_value = :value, subject_id = :subject_id) "
"WHERE id = :id").arg(table));
query.bindValue(":id", id);
query.bindValue(":key", attribute.first);
query.bindValue(":value", attribute.second);
query.bindValue(":subject_id", subject->idToStringWithoutBraces());
query.exec();
}
/* else
{
query.prepare(QString("INSERT INTO %1 (id, attribute_key, attribute_value, subject_id)"
"VALUES (:id, :key, :value, :subject_id)").arg(table));
query.bindValue(":id", id);
query.bindValue(":key", attribute.first);
query.bindValue(":value", attribute.second);
query.bindValue(":subject_id", subject->idToStringWithoutBraces());
query.exec();
}*/
}
}
void DatabaseManager::deleteItem(QString id, QString table)
{
query.prepare(QString("DELETE FROM %1 WHERE id = :id").arg(table));
query.bindValue(":id", id);
query.exec();
}
QString DatabaseManager::getTestMessage(int id)
@ -80,7 +121,6 @@ int DatabaseManager::createTestRecord(QString msg)
query.bindValue(":id", id);
query.bindValue(":msg", msg);
query.exec();
query.next();
qDebug() << "U inserted new row, id: " << id << "msg: " << getTestMessage(id);
return id;
}
@ -88,9 +128,61 @@ int DatabaseManager::createTestRecord(QString msg)
bool DatabaseManager::checkIfExists(QString id, QString table)
{
query.prepare(QString("SELECT exists(SELECT 1 FROM %1 WHERE id = :id)").arg(table));
query.prepare(QString("SELECT EXISTS(SELECT 1 FROM %1 WHERE id = :id)").arg(table));
query.bindValue(":id", id);
query.exec();
query.next();
query.first();
return query.value(0).toBool();
}
void DatabaseManager::dropTables()
{
query.exec("DROP TABLE IF EXISTS colnod_token, colnod_subject, colnod_subject_attribute");
}
void DatabaseManager::createTables()
{
query.exec("CREATE TABLE colnod_subject ("
"id CHAR(36) primary key,"
"name VARCHAR(255),"
"last_update timestamp)");
query.exec("CREATE TABLE colnod_token ("
"id CHAR(36) primary key,"
"token_data VARCHAR(255),"
"subject_id CHAR(36),"
"last_update timestamp)");
query.exec("CREATE TABLE colnod_subject_attribute ("
"id VARCHAR(255) primary key,"
"attribute_key VARCHAR(255),"
"attribute_value VARCHAR(255),"
"subject_id CHAR(36))");
query.exec("CREATE TABLE users ("
"id VARCHAR(255) primary key,"
"email VARCHAR(255),"
"token_data VARCHAR(255),"
"last_update timestamp)");
}
uint64_t DatabaseManager::getLastUpdate()
{
query.exec("SELECT EXTRACT(EPOCH FROM (SELECT MIN(last_sync) FROM changes))*1000");
query.next();
return query.value(0).toULongLong();
}
void DatabaseManager::makeTestingLast_sync()
{
query.prepare("INSERT INTO last_changes (last_sync) "
"VALUES (:time)");
query.bindValue(":time", QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.exec();
}
void DatabaseManager::updateLastSync()
{
query.prepare("INSERT INTO changes (last_sync) "
"VALUES (:time)");
query.bindValue(":time", QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz"));
query.exec();
}

12
src/colnod/databasemanager.h
View File

@ -18,15 +18,23 @@ class DatabaseManager : public QObject
public:
DatabaseManager(QString host, QString dbName, QString user, QString password);
bool saveSubject(Subject* subject, QString table);
bool saveToken(Token* token, QString table);
void saveSubject(Subject* subject, QString table);
void saveToken(Token* token, QString table);
void saveSubjectAttribute(Subject* subject, QString table);
void deleteItem(QString id, QString table);
QString getTestMessage(int id);
int createTestRecord(QString msg);
//
void makeTestingLast_sync();
//
void dropTables();
void createTables();
bool getOpened() {return opened;}
uint64_t getLastUpdate();
void updateLastSync();
private:

181
src/colnod/datamanager.cpp
View File

@ -3,16 +3,26 @@
DataManager::DataManager()
{
db = new DatabaseManager("192.168.1.94", "deloitte", "postgres", "34rjkciea12");
//db = new DatabaseManager("localhost", "deloitte", "postgres", "34rjkciea12");
}
void DataManager::localSaveSubjects(QString response)
{
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
for (QJsonValueRef item : items)
if (items.size() > 0)
{
auto subject = new Subject();
subject->fromJSON(item.toObject());
for (QJsonValueRef item : items)
{
auto subject = new Subject();
subject->fromJSON(item.toObject());
subjects.append(subject);
}
}
else
{
Subject* subject = new Subject;
subject->setName("empty");
subjects.append(subject);
}
}
@ -21,42 +31,163 @@ void DataManager::localSaveTokens(QString response)
{
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
for (QJsonValueRef item : items)
if (items.size() > 0)
{
auto token = new Token();
token->fromJSON(item.toObject());
for (QJsonValueRef item : items)
{
auto token = new Token();
token->fromJSON(item.toObject());
tokens.append(token);
}
}
else
{
Token* token = new Token;
token->setName("empty");
tokens.append(token);
}
}
void DataManager::pushDataToDB()
void DataManager::localSaveSubjectsToDelete(QString json)
{
QSqlDatabase::database().transaction();
qDebug() << "In progress...";
int counter = 0;
QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
if (items.size() > 0)
{
for (QJsonValueRef item : items)
{
subjectsToDelete.append(item.toString());
}
}
else
{
subjectsToDelete.append("empty");
}
}
void DataManager::localSaveTokensToDelete(QString json)
{
QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
if (items.size() > 0)
{
for (QJsonValueRef item : items)
{
tokensToDelete.append(item.toString());
}
}
else
{
tokensToDelete.append("empty");
}
}
void DataManager::pushSubjectsToDB()
{
if (subjects.size() == 1 && subjects[0]->getName() == "empty")
{
qDebug() << "All subjects are up to date";
return;
}
//QSqlDatabase::database().transaction();
qDebug() << "Subjects in progress...";
for (auto subject : subjects)
{
if (db->saveSubject(subject, "subject"))
counter++;
db->saveSubjectAttribute(subject, "subject_attribute");
db->saveSubject(subject, "colnod_subject");
db->saveSubjectAttribute(subject, "colnod_subject_attribute");
}
qDebug() << counter << " subjects has been updated";
counter = 0;
for (auto token : tokens)
{
if (db->saveToken(token, "token"))
counter++;
}
QSqlDatabase::database().commit();
qDebug() << counter << " tokens has been updated";
//QSqlDatabase::database().commit();
qDebug() << subjects.size() << " subjects has been updated";
}
bool DataManager::haveData() {
void DataManager::pushTokensToDB()
{
if (tokens.size() == 1 && tokens[0]->getName() == "empty")
{
qDebug() << "All tokens are up to date";
return;
}
//QSqlDatabase::database().transaction();
qDebug() << "Tokens in progress...";
for (auto token : tokens)
{
db->saveToken(token, "colnod_token");
}
//QSqlDatabase::database().commit();
qDebug() << tokens.size() << " tokens has been updated";
}
void DataManager::deleteSubjectsFromDB()
{
if (subjectsToDelete.size() == 1 && subjectsToDelete[0] == "empty")
{
qDebug() << "No subjects to be deleted";
return;
}
qDebug() << "Deleted subjects in progress...";
//QSqlDatabase::database().transaction();
for (auto subjectId : subjectsToDelete)
{
db->deleteItem(subjectId, "colnod_subject");
}
//QSqlDatabase::database().commit();
qDebug() << subjectsToDelete.size() << "subject has beed deleted";
}
void DataManager::deleteTokensFromDB()
{
if (tokensToDelete.size() == 1 && tokensToDelete[0] == "empty")
{
qDebug() << "No tokens to be deleted";
return;
}
// QSqlDatabase::database().transaction();
qDebug() << "Deleted tokens in progress...";
for (auto tokenId : tokensToDelete)
{
db->deleteItem(tokenId, "colnod_token");
}
//QSqlDatabase::database().commit();
qDebug() << tokensToDelete.size() << "tokens has beed deleted";
}
bool DataManager::haveData()
{
if ( !subjects.isEmpty() && !tokens.isEmpty() )
return true;
return false;
}
bool DataManager::haveChangedData()
{
if ( !subjects.isEmpty() && !tokens.isEmpty() && !subjectsToDelete.isEmpty() && !tokensToDelete.isEmpty() )
return true;
return false;
}
uint64_t DataManager::getLastUpdate()
{
return db->getLastUpdate();
}
void DataManager::emptyDatabase()
{
db->dropTables();
db->createTables();
}
void DataManager::clear()
{
subjects.clear();
subjectsToDelete.clear();
tokens.clear();
tokensToDelete.clear();
}
void DataManager::updateLastSync()
{
db->updateLastSync();
}

22
src/colnod/datamanager.h
View File

@ -19,19 +19,29 @@ public:
void localSaveSubjects(QString jsonResponse);
void localSaveTokens(QString jsonResponse);
void pushDataToDB();
void localSaveSubjectsToDelete(QString jsonResponse);
void localSaveTokensToDelete(QString jsonResponse);
void pushSubjectsToDB();
void pushTokensToDB();
void deleteSubjectsFromDB();
void deleteTokensFromDB();
void updateLastSync();
uint64_t getLastUpdate();
bool haveData();
bool haveChangedData();
void emptyDatabase();
void clear();
private:
DatabaseManager *db = nullptr;
QList<QPointer<Subject>> subjects;
QList<QPointer<Token>> tokens;
//Data *dataToProcess = nullptr;
QList<QString> subjectsToDelete;
QList<QString> tokensToDelete;
};
#endif // DATAMANAGER_H

2
src/colnod/requestmanager.h
View File

@ -84,7 +84,7 @@ private:
QJsonDocument config;
QString authHeader;
QString host = "https://192.168.1.3:8443";
//QString host = "http://localhost:8080";
// QString host = "https://localhost:8080";
QString password = "spaceti1";
QString user = "spaceti";

2
test/unitTests/UnitTest.cpp
View File

@ -46,8 +46,8 @@ void UnitTest::getSubjects()
void UnitTest::getTokens()
{
RequestManager* testAccess = new RequestManager();
QSignalSpy spy(testAccess, SIGNAL(responseReady()));
Response* response = testAccess->getTokens();
QSignalSpy spy(response, &Response::ready);
QCOMPARE(spy.count(), 0);
spy.wait(15000);
QVERIFY(spy.isValid());

28
test/unitTests/complextest.cpp
View File

@ -4,6 +4,7 @@ void ComplexTest::initTestCase()
{
reqMan = new RequestManager;
start = QDateTime::currentMSecsSinceEpoch()-2000;
api = new ColnodAPI;
qDebug() << start;
}
@ -99,6 +100,33 @@ void ComplexTest::complexTest()
}
void ComplexTest::fullSync()
{
api->prepareDatabase();
api->downloadData();
QSignalSpy spy(api, &ColnodAPI::dataDownloaded);
spy.wait(10000);
api->syncDataFromColnod();
// api->syncDataToColnod();
}
void ComplexTest::syncLastUpdated()
{
api->downloadChangedDataFromColnod();
QSignalSpy spy(api, &ColnodAPI::dataDownloaded);
spy.wait(10000);
api->syncChangedDataFromColnod();
//api->syncUpdatedDataToColnod();
}
void ComplexTest::testingStuff()
{
//DatabaseManager* db = new DatabaseManager("192.168.1.94", "deloitte", "postgres", "34rjkciea12");
//DatabaseManager* db = new DatabaseManager("localhost", "deloitte", "postgres", "34rjkciea12");
//db->makeTestingLast_sync();
}
// HELPFUL_FUNCTIONS
int ComplexTest::getNumOfSubjects()
{

6
test/unitTests/complextest.h
View File

@ -5,6 +5,7 @@
#include <QtTest/QtTest>
#include "response.h"
#include "requestmanager.h"
#include "colnodapi.h"
class ComplexTest : public QObject
{
@ -13,6 +14,9 @@ class ComplexTest : public QObject
private slots:
void initTestCase();
void complexTest();
void fullSync();
void syncLastUpdated();
void testingStuff();
void cleanupTestCase();
private:
@ -29,6 +33,6 @@ private:
QString password = "spaceti1";
RequestManager* reqMan = nullptr;
uint64_t start = 0;
ColnodAPI* api = nullptr;
};