Changing stuff for one table sync

This commit is contained in:
Ondřej Fischer 2019-10-21 10:06:49 +02:00
parent dcd0034110
commit ab6a200551
13 changed files with 264 additions and 161 deletions

View File

@ -8,12 +8,27 @@ int main(int argc, char *argv[])
QCoreApplication a(argc, argv);
ColnodAPI *api = new ColnodAPI();
api->downloadData(); // this will start downloading data
/*api->downloadData(); // this will start downloading data
QObject::connect(api, &ColnodAPI::dataDownloaded, [=](){
qDebug() << "now I have all data";
api->syncDataToColnod();
QCoreApplication::quit();
});
});*/
// bool syncChanged = false;
// api->downloadChangedDataFromColnod();
// QObject::connect(api, &ColnodAPI::downloadData, [&](){
// qDebug() << "Data ready.";
// api->syncChangedDataFromColnod();
// if (!syncChanged)
// {
// api->syncChangedDataToColnod();
// syncChanged = true;
// }
// });
api->loadDataFromColnod();
api->loadDataFromDB();
return a.exec();
}

View File

@ -8,87 +8,103 @@ ColnodAPI::ColnodAPI()
// PUBLIC FUNCTIONS
void ColnodAPI::downloadData()
//void ColnodAPI::downloadData()
//{
// Response* resp0 = reqMan->getSubjects();
// connect(resp0, &Response::ready, [=](){
// dataMan->localSaveSubjects(resp0->getJson());
// areDataReady();
// resp0->deleteLater();
// });
// Response* resp1 = reqMan->getTokens();
// connect(resp1, &Response::ready, [=](){
// dataMan->localSaveTokens(resp1->getJson());
// areDataReady();
// resp1->deleteLater();
// });
// /*
// QNetworkReply* reply2 = reqMan->getTokens();
// connect(reply2, &QNetworkReply::finished, [=](){
// data->saveTokens(reply2->readAll());
// });
// */
//}
void ColnodAPI::loadDataFromColnod()
{
Response* resp0 = reqMan->getSubjects();
connect(resp0, &Response::ready, [=](){
dataMan->localSaveSubjects(resp0->getJson());
areDataReady();
resp0->deleteLater();
});
Response* resp1 = reqMan->getTokens();
connect(resp1, &Response::ready, [=](){
dataMan->localSaveTokens(resp1->getJson());
areDataReady();
resp1->deleteLater();
});
/*
QNetworkReply* reply2 = reqMan->getTokens();
connect(reply2, &QNetworkReply::finished, [=](){
data->saveTokens(reply2->readAll());
});
*/
loadTokensFromColnod();
loadUpdatedDataFromColnod();
loadDeletedDataFromColnod();
}
void ColnodAPI::downloadChangedDataFromColnod()
void ColnodAPI::loadTokensFromColnod()
{
downloadUpdatedData();
downloadDeletedData();
Response* resp = reqMan->getTokens();
connect(resp, &Response::ready, [=](){
dataMan->fillFullTokensColnod(resp->getJson());
});
}
void ColnodAPI::downloadUpdatedData()
void ColnodAPI::loadUpdatedDataFromColnod()
{
uint64_t lastUpdate = dataMan->getLastUpdate();
qDebug() << lastUpdate;
uint64_t lastUpdate = dataMan->getLastSync();
Response* resp0 = reqMan->getSubjectsUpdated(lastUpdate);
connect(resp0, &Response::ready, [=](){
dataMan->localSaveSubjects(resp0->getJson());
dataMan->fillSubjectsColnod(resp0->getJson());
areChangedDataReady();
resp0->deleteLater();
});
Response* resp1 = reqMan->getTokensUpdated(lastUpdate);
connect(resp1, &Response::ready, [=](){
dataMan->localSaveTokens(resp1->getJson());
dataMan->fillTokensColnod(resp1->getJson());
areChangedDataReady();
resp1->deleteLater();
});
}
void ColnodAPI::downloadDeletedData()
void ColnodAPI::loadDeletedDataFromColnod()
{
uint64_t lastUpdate = dataMan->getLastUpdate();
uint64_t lastUpdate = dataMan->getLastSync();
Response* resp2 = reqMan->getSubjectsDeleted(lastUpdate);
connect(resp2, &Response::ready, [=](){
dataMan->localSaveSubjectsToDelete(resp2->getJson());
dataMan->fillSubjectsToDeleteColnod(resp2->getJson());
areChangedDataReady();
resp2->deleteLater();
});
Response* resp3 = reqMan->getTokensDeleted(lastUpdate);
connect(resp3, &Response::ready, [=](){
dataMan->localSaveTokensToDelete(resp3->getJson());
dataMan->fillTokensToDeleteColnod(resp3->getJson());
areChangedDataReady();
resp3->deleteLater();
});
}
void ColnodAPI::syncDataFromColnod()
void ColnodAPI::loadDataFromDB()
{
dataMan->pushTokensToDB();
dataMan->pushSubjectsToDB();
dataMan->updateLastSync();
dataMan->clear();
dataMan->fillTokensDB();
dataMan->fillTokensToDeleteDB();
}
void ColnodAPI::syncChangedDataFromColnod()
//void ColnodAPI::syncDataFromColnod()
//{
// dataMan->pushTokensToDB();
// dataMan->pushSubjectsToDB();
// dataMan->updateLastSync();
// dataMan->clear();
//}
void ColnodAPI::syncDataFromColnod()
{
syncUpdatedDataFromColnod();
syncDeletedDataFromColnod();
dataMan->updateLastSync();
//dataMan->updateLastSync();
dataMan->clear();
}
@ -105,22 +121,12 @@ void ColnodAPI::syncDeletedDataFromColnod()
dataMan->deleteTokensFromDB();
}
void ColnodAPI::syncDataToColnod()
{
// TODO
}
void ColnodAPI::syncChangedDataToColnod()
void ColnodAPI::syncDataFromDB()
{
// TODO
dataMan->loadTokensToAdd();
dataMan->loadTokensToDelete();
processDataToColnod();
dataMan->clear();
}
@ -129,22 +135,23 @@ void ColnodAPI::processDataToColnod()
QList<QString>* tokensToDelete = dataMan->getTokensToDelete();
if ((*tokensToDelete)[0] != "empty")
{
Response* resp0 = reqMan->deleteTokens(dataMan->getTokensToDelete());
Response* resp0 = reqMan->deleteTokens(tokensToDelete);
connect(resp0, &Response::ready, [=](){
areProcessDataReady();
isProcessDone();
resp0->deleteLater();
});
}
QList<QPointer<Token>>* tokens = dataMan->getTokens();
if ((*tokens)[0]->getName() != "empty")
QHash<QString, QPointer<Token>>* tokens = dataMan->getTokens();
if (!tokens[0].contains("empty"))
{
Response* resp1 = reqMan->setTokens(dataMan->getTokens());
Response* resp1 = reqMan->setTokens(tokens);
connect(resp1, &Response::ready, [=](){
areProcessDataReady();
isProcessDone();
resp1->deleteLater();
});
}
connect(this, &ColnodAPI::dataLoaded, this, &ColnodAPI::syncChangedDataFromColnod);
dataMan->clear();
connect(this, &ColnodAPI::dataProcessed, this, &ColnodAPI::loadDataFromColnod);
}
void ColnodAPI::prepareDatabase()
@ -166,10 +173,10 @@ void ColnodAPI::areChangedDataReady()
}
void ColnodAPI::areProcessDataReady()
void ColnodAPI::isProcessDone()
{
if (dataMan->haveProcessData())
{
emit dataLoaded();
emit dataProcessed();
}
}

View File

@ -14,14 +14,14 @@ class ColnodAPI : public QObject
public:
ColnodAPI();
void downloadData();
///
//void downloadData();
//void syncDataFromColnod();
//void syncDataToColnod();
///
void syncDataFromColnod();
void syncDataToColnod();
void syncChangedDataFromColnod();
void syncChangedDataToColnod();
void syncDataFromDB();
@ -29,24 +29,30 @@ public:
void prepareDatabase();
public slots:
void downloadChangedDataFromColnod();
void loadDataFromColnod();
void loadDataFromDB();
signals:
void dataDownloaded();
void dataLoaded();
void dataProcessed();
private slots:
private:
void downloadUpdatedData();
void downloadDeletedData();
void loadTokensFromColnod();
void loadUpdatedDataFromColnod();
void loadDeletedDataFromColnod();
void loadUpdatedDataFromDB();
void loadDeletedDataFromDB();
void syncUpdatedDataFromColnod();
void syncDeletedDataFromColnod();
void areDataReady();
void areChangedDataReady();
void areProcessDataReady();
void isProcessDone();
void processDataToColnod();

View File

@ -164,11 +164,13 @@ void DatabaseManager::createTables()
"last_update timestamp)");
}
uint64_t DatabaseManager::getLastUpdate()
uint64_t DatabaseManager::getAndSetLastSync()
{
query.exec("SELECT EXTRACT(EPOCH FROM (SELECT MIN(last_sync) FROM changes))*1000"); //MAX for using, MIN for testing
query.exec("SELECT EXTRACT(EPOCH FROM (SELECT last_sync FROM config))*1000");
query.next();
return query.value(0).toULongLong();
uint64_t temp = query.value(0).toULongLong();
query.exec("UPDATE config SET last_sync = NOW()");
return temp;
}
void DatabaseManager::makeTestingLast_sync()

View File

@ -18,6 +18,8 @@ class DatabaseManager : public QObject
public:
DatabaseManager(QString host, QString dbName, QString user, QString password);
void saveSubject(Subject* subject, QString table);
void saveToken(Token* token, QString table);
void saveSubjectAttribute(Subject* subject, QString table);
@ -33,7 +35,7 @@ public:
void createTables();
bool getOpened() {return opened;}
uint64_t getLastUpdate();
uint64_t getAndSetLastSync();
void updateLastSync();
QSqlQuery* getTokensToAdd();

View File

@ -3,31 +3,11 @@
DataManager::DataManager()
{
db = new DatabaseManager("192.168.1.94", "deloitte", "postgres", "34rjkciea12");
lastSync = db->getAndSetLastSync();
//db = new DatabaseManager("localhost", "deloitte", "postgres", "34rjkciea12");
}
void DataManager::localSaveSubjects(QString response)
{
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
if (items.size() > 0)
{
for (QJsonValueRef item : items)
{
auto subject = new Subject();
subject->fromJSON(item.toObject());
subjects.append(subject);
}
}
else
{
Subject* subject = new Subject;
subject->setName("empty");
subjects.append(subject);
}
}
void DataManager::localSaveTokens(QString response)
void DataManager::fillFullTokensColnod(QString response)
{
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
@ -37,18 +17,68 @@ void DataManager::localSaveTokens(QString response)
{
auto token = new Token();
token->fromJSON(item.toObject());
tokens.append(token);
fullTokensColnod.insert(token->idToStringWithoutBraces(), token);
}
}
else
{
Token* token = new Token;
token->setName("empty");
tokens.append(token);
auto token = new Token("empty");
fullTokensColnod.insert(token->idToStringWithoutBraces(), token);
}
}
void DataManager::localSaveSubjectsToDelete(QString json)
void DataManager::fillTokensColnod(QString response)
{
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
if (items.size() > 0)
{
for (QJsonValueRef item : items)
{
auto token = new Token();
token->fromJSON(item.toObject());
tokensColnod.insert(token->idToStringWithoutBraces(), token);
}
}
else
{
auto token = new Token("empty");
tokensColnod.insert(token->idToStringWithoutBraces(), token);
}
}
void DataManager::fillSubjectsColnod(QString response)
{
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
if (items.size() > 0)
{
for (QJsonValueRef item : items)
{
auto subject = new Subject();
subject->fromJSON(item.toObject());
subjectsColnod.insert(subject->idToStringWithoutBraces(), subject);
}
}
else
{
Subject* subject = new Subject("empty");
subjectsColnod.insert(subject->idToStringWithoutBraces(), subject);
}
}
void DataManager::fillTokensDB()
{
QSqlQuery* query = db->getUpdatedTokens(lastSync);
// TODO
}
void DataManager::fillSubjectsDB()
{
// Not needed yet
}
void DataManager::fillSubjectsToDeleteColnod(QString json)
{
QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
@ -56,16 +86,17 @@ void DataManager::localSaveSubjectsToDelete(QString json)
{
for (QJsonValueRef item : items)
{
subjectsToDelete.append(item.toString());
subjectsToDeleteColnod.append(item.toString());
}
}
else
{
subjectsToDelete.append("empty");
subjectsToDeleteColnod.append("empty");
}
}
void DataManager::localSaveTokensToDelete(QString json)
void DataManager::fillTokensToDeleteColnod(QString json)
{
QJsonDocument doc = QJsonDocument::fromJson(json.toUtf8());
QJsonArray items = doc.object()["items"].toArray();
@ -73,111 +104,117 @@ void DataManager::localSaveTokensToDelete(QString json)
{
for (QJsonValueRef item : items)
{
tokensToDelete.append(item.toString());
tokensToDeleteColnod.append(item.toString());
}
}
else
{
tokensToDelete.append("empty");
tokensToDeleteColnod.append("empty");
}
}
void DataManager::fillSubjectsToDeleteDB()
{
// Not needed yet
}
void DataManager::fillTokensToDeleteDB()
{
// TODO
// get data from db, maybe it's better to get all tokens together then filter deleted
}
void DataManager::pushSubjectsToDB()
{
if (subjects.size() == 1 && subjects[0]->getName() == "empty")
if (subjectsColnod.contains("empty"))
{
qDebug() << "All subjects are up to date";
return;
}
//QSqlDatabase::database().transaction();
qDebug() << "Subjects in progress...";
for (auto subject : subjects)
for (auto subject : subjectsColnod)
{
db->saveSubject(subject, "colnod_subject");
db->saveSubjectAttribute(subject, "colnod_subject_attribute");
}
//QSqlDatabase::database().commit();
qDebug() << subjects.size() << " subjects has been updated";
qDebug() << subjectsColnod.size() << " subjects has been updated";
}
void DataManager::pushTokensToDB()
{
if (tokens.size() == 1 && tokens[0]->getName() == "empty")
if (tokensColnod.contains("empty"))
{
qDebug() << "All tokens are up to date";
return;
}
//QSqlDatabase::database().transaction();
qDebug() << "Tokens in progress...";
for (auto token : tokens)
for (auto token : tokensColnod)
{
db->saveToken(token, "colnod_token");
}
//QSqlDatabase::database().commit();
qDebug() << tokens.size() << " tokens has been updated";
qDebug() << tokensColnod.size() << " tokens has been updated";
}
void DataManager::deleteSubjectsFromDB()
{
if (subjectsToDelete.size() == 1 && subjectsToDelete[0] == "empty")
if (subjectsToDeleteColnod[0] == "empty")
{
qDebug() << "No subjects to be deleted";
return;
}
qDebug() << "Deleted subjects in progress...";
//QSqlDatabase::database().transaction();
for (auto subjectId : subjectsToDelete)
for (auto subjectId : subjectsToDeleteColnod)
{
db->deleteItem(subjectId, "colnod_subject");
}
//QSqlDatabase::database().commit();
qDebug() << subjectsToDelete.size() << "subject has beed deleted";
qDebug() << subjectsToDeleteColnod.size() << "subject has beed deleted";
}
void DataManager::deleteTokensFromDB()
{
if (tokensToDelete.size() == 1 && tokensToDelete[0] == "empty")
if (tokensToDeleteColnod[0] == "empty")
{
qDebug() << "No tokens to be deleted";
return;
}
// QSqlDatabase::database().transaction();
qDebug() << "Deleted tokens in progress...";
for (auto tokenId : tokensToDelete)
for (auto tokenId : tokensToDeleteColnod)
{
db->deleteItem(tokenId, "colnod_token");
}
//QSqlDatabase::database().commit();
qDebug() << tokensToDelete.size() << "tokens has beed deleted";
qDebug() << tokensToDeleteColnod.size() << "tokens has beed deleted";
}
bool DataManager::haveData()
{
if ( !subjects.isEmpty() && !tokens.isEmpty() )
if ( !subjectsColnod.isEmpty() && !tokensColnod.isEmpty() )
return true;
return false;
}
bool DataManager::haveChangedData()
{
if ( !subjects.isEmpty() && !tokens.isEmpty() && !subjectsToDelete.isEmpty() && !tokensToDelete.isEmpty() )
if ( !subjectsColnod.isEmpty() && !tokensColnod.isEmpty() && !subjectsToDeleteColnod.isEmpty() && !tokensToDeleteColnod.isEmpty() )
return true;
return false;
}
bool DataManager::haveProcessData()
{
if ( !tokens.isEmpty() && !tokensToDelete.isEmpty())
if ( !tokensColnod.isEmpty() && !tokensToDeleteColnod.isEmpty())
return true;
return false;
}
uint64_t DataManager::getLastUpdate()
{
return db->getLastUpdate();
}
void DataManager::emptyDatabase()
{
db->dropTables();
@ -186,10 +223,14 @@ void DataManager::emptyDatabase()
void DataManager::clear()
{
subjects.clear();
subjectsToDelete.clear();
tokens.clear();
tokensToDelete.clear();
subjectsColnod.clear();
subjectsToDeleteColnod.clear();
tokensColnod.clear();
tokensToDeleteColnod.clear();
subjectsDB.clear();
subjectsToDeleteDB.clear();
tokensDB.clear();
tokensToDeleteDB.clear();
}
void DataManager::updateLastSync()
@ -204,10 +245,10 @@ void DataManager::loadTokensToAdd()
{
Token* token = new Token(query->value(0).toString(), query->value(1).toString());
token->setSubjectId(db->getId(token->getName()));
tokens.append(token);
tokensDB.insert(token->idToStringWithoutBraces(), token);
}
if (tokens.isEmpty())
tokens.append(new Token("empty"));
if (tokensDB.isEmpty())
tokensDB.insert("empty", nullptr);
}
void DataManager::loadTokensToDelete()
@ -215,10 +256,10 @@ void DataManager::loadTokensToDelete()
QSqlQuery* query = db->getTokensToDelete();
while (query->next())
{
tokensToDelete.append(query->value(0).toString());
tokensToDeleteDB.append(query->value(0).toString());
}
if (tokensToDelete.isEmpty())
tokensToDelete.append("empty");
if (tokensToDeleteDB.isEmpty())
tokensToDeleteDB.append("empty");
}

View File

@ -17,10 +17,18 @@ class DataManager : public QObject
public:
DataManager();
void localSaveSubjects(QString jsonResponse);
void localSaveTokens(QString jsonResponse);
void localSaveSubjectsToDelete(QString jsonResponse);
void localSaveTokensToDelete(QString jsonResponse);
void getAndSetLastSync();
void fillFullTokensColnod(QString jsonResponse);
void fillTokensColnod(QString jsonResponse);
void fillSubjectsColnod(QString jsonResponse);
void fillSubjectsToDeleteColnod(QString jsonResponse);
void fillTokensToDeleteColnod(QString jsonResponse);
void fillTokensDB();
void fillSubjectsDB();
void fillSubjectsToDeleteDB();
void fillTokensToDeleteDB();
void pushSubjectsToDB();
void pushTokensToDB();
@ -28,7 +36,6 @@ public:
void deleteTokensFromDB();
void updateLastSync();
uint64_t getLastUpdate();
bool haveData();
bool haveChangedData();
@ -40,16 +47,28 @@ public:
void loadTokensToAdd();
void loadTokensToDelete();
QList<QPointer<Token>>* getTokens() {return &tokens;}
QList<QString>* getTokensToDelete() {return &tokensToDelete;}
QHash<QString, QPointer<Token>>* getTokens() {return &tokensColnod;}
QList<QString>* getTokensToDelete() {return &tokensToDeleteColnod;}
const uint64_t &getLastSync() {return lastSync;}
private:
DatabaseManager *db = nullptr;
QList<QPointer<Subject>> subjects;
QList<QPointer<Token>> tokens;
QList<QString> subjectsToDelete;
QList<QString> tokensToDelete;
uint64_t lastSync;
QHash<QString, QPointer<Subject>> subjectsColnod;
QHash<QString, QPointer<Token>> tokensColnod;
QHash<QString, QPointer<Token>> fullTokensColnod;
QHash<QString, QPointer<Subject>> subjectsDB; // in future
QHash<QString, QPointer<Token>> tokensDB;
QList<QString> subjectsToDeleteColnod;
QList<QString> tokensToDeleteColnod;
QList<QString> subjectsToDeleteDB;
QList<QString> tokensToDeleteDB;
};
#endif // DATAMANAGER_H

View File

@ -157,7 +157,7 @@ QByteArray RequestManager::makeTimeStampJson(uint64_t time)
return doc.toJson();
}
QByteArray RequestManager::makeTokensJson(QList<QPointer<Token>>* tokens)
QByteArray RequestManager::makeTokensJson(QHash<QString, QPointer<Token> > *tokens)
{
QJsonObject items;
QJsonArray jsonTokens;
@ -265,7 +265,7 @@ Response* RequestManager::getAccessGroupMembershipDeleted(uint64_t time)
return postRequestReturnLogic("/AccessManager/getAccessGroupMembershipDeleted", makeTimeStampJson(time));
}
Response* RequestManager::setTokens(QList<QPointer<Token>>* tokens)
Response* RequestManager::setTokens(QHash<QString, QPointer<Token> > *tokens)
{
return postRequestReturnLogic("/TokenManager/setTokens", makeTokensJson(tokens));
}

View File

@ -49,7 +49,7 @@ public:
Response* getAccessGroupMembershipUpdated(uint64_t time);
Response* getAccessGroupMembershipDeleted(uint64_t time);
Response* setTokens(QList<QPointer<Token>>* tokens);
Response* setTokens(QHash<QString, QPointer<Token>>* tokens);
Response* deleteTokens(QList<QString>* ids);
Response* login(); //I want it private, but unittest???
@ -65,7 +65,7 @@ private:
QString getLoginHeader(QString username, QString hash);
QByteArray makeTimeStampJson(uint64_t time);
QByteArray makeTokensJson(QList<QPointer<Token>>* tokens);
QByteArray makeTokensJson(QHash<QString, QPointer<Token>>* tokens);
QByteArray makeIdsJson(QList<QString>* ids);
Response* getLoginResponse();

View File

@ -4,6 +4,11 @@
#include <QJsonArray>
Subject::Subject(QString namee)
{
this->name = namee;
}
QString Subject::getEmail() const {
return attributes["email"].toString();
}

View File

@ -13,6 +13,8 @@ class Subject : public Entity {
public:
using Entity::Entity;
Subject(QString name);
QString getEmail() const;
const QJsonObject &getAttributes() const;

View File

@ -212,13 +212,13 @@ void UnitTest::setTokens()
QJsonObject obj = doc.object();
QJsonArray arr = obj["items"].toArray();
qDebug() << arr;
QList<QPointer<Token>> tokens;
QHash<QString, QPointer<Token>> tokens;
for (auto object : arr)
{
auto token = new Token;
token->fromJSON(object.toObject());
qDebug() << token;
tokens.append(token);
tokens.insert(token->idToStringWithoutBraces(), token);
}
RequestManager* testAccess = new RequestManager();
QSignalSpy spy(testAccess, SIGNAL(responseReady()));

View File

@ -43,8 +43,9 @@ void ComplexTest::complexTest()
// ADD_TOKENS_CHECK_BEGIN
QList<QPointer<Token>> tokens;
tokens.append(getTestToken());
QHash<QString, QPointer<Token>> tokens;
auto token = getTestToken();
tokens.insert(token->idToStringWithoutBraces(), token);
Response* setTokensResp = reqMan->setTokens(&tokens);
QSignalSpy setTokensRespSpy(setTokensResp, &Response::ready);
QCOMPARE(setTokensRespSpy.count(), 0);
@ -112,11 +113,14 @@ void ComplexTest::fullSync()
void ComplexTest::syncLastUpdated()
{
api->downloadChangedDataFromColnod();
api->loadDataFromColnod();
QSignalSpy spy(api, &ColnodAPI::dataDownloaded);
spy.wait(10000);
api->syncChangedDataFromColnod();
api->syncDataFromColnod();
// api->syncChangedDataToColnod();
spy.wait(10000);
api->syncDataFromColnod();
}