ondra/colnod-connector
1
0
Fork 1
mirror of https://gitlab.com/spaceti-app/integrations/acs/colnod-connector.git synced 2026-07-12 14:30:50 +02:00
Code Issues Projects Releases Wiki Activity

Finished Info output

Browse Source
This commit is contained in:
Ondřej Fischer 2019-11-21 10:39:55 +01:00
parent 2ce5a025a9
commit 80c550ff03
8 changed files with 121 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/colnod/colnodapi.cpp
View File

@ -126,6 +126,7 @@ void ColnodAPI::setTokens(const QHash<QString, std::shared_ptr<Token>> &tokens)
emit partOfDataProcessFinished(); emit partOfDataProcessFinished();
} }
else { else {
qWarning(colnod) << "Tokens wasn't updated";
// TODO: error: data wasnt upload to colnod // TODO: error: data wasnt upload to colnod
} }
}); });
@ -141,6 +142,7 @@ void ColnodAPI::deleteTokens(const QList<QString> &ids)
emit partOfDataProcessFinished(); emit partOfDataProcessFinished();
} }
else { else {
qWarning(colnod) << "Tokens wasn't deleted";
// TODO: error: data wasnt upload to colnod // TODO: error: data wasnt upload to colnod
} }
}); });
@ -188,12 +190,12 @@ void ColnodAPI::areDataReady()
{ {
if (++amountData == 4) { if (++amountData == 4) {
qDebug(colnod) << "COLNOD DATA:"; qInfo(colnod) << TAB << "-> COLNOD DATA:";
qDebug(colnod) << TAB << data->updatedTokens.size() << "TOKENS TO UPDATE"; qInfo(colnod) << TAB << TAB << data->updatedTokens.size() << "TOKENS WERE MODIFIED";
qDebug(colnod) << TAB << data->deletedTokens.size() << "TOKENS TO DELETE"; qInfo(colnod) << TAB << TAB << data->deletedTokens.size() << "TOKENS MARKED FOR DELETION";
qDebug(colnod) << TAB << data->updatedSubjects.size() << "SUBJECTS TO UPDATE"; qInfo(colnod) << TAB << TAB << data->updatedSubjects.size() << "SUBJECTS WERE MODIFIED";
qDebug(colnod) << TAB << data->deletedSubjects.size() << "SUBJECTS TO DELETE"; qInfo(colnod) << TAB << TAB << data->deletedSubjects.size() << "SUBJECTS MARKED FOR DELETION";
qDebug(colnod) << UNDERSCORE << endl; qInfo(colnod) << DASH;
emit downloadDataFinished(data); emit downloadDataFinished(data);
newData(); newData();
@ -202,8 +204,14 @@ void ColnodAPI::areDataReady()
void ColnodAPI::isDataProcessFinished() void ColnodAPI::isDataProcessFinished()
{ {
if (++amountProccessesFinished == 2) if (++amountProccessesFinished == 2) {
qInfo(colnod) << TAB << "-> DATA PROCESSED TO COLNOD:";
qInfo(colnod) << TAB << TAB << data->updatedTokens.size() << "TOKENS UPDATED";
qInfo(colnod) << TAB << TAB << data->deletedTokens.size() << "TOKENS DELETED";
qInfo(colnod) << DASH;
emit dataProcessFinished(); emit dataProcessFinished();
}
} }
void ColnodAPI::saveUpdatedTokens(const QByteArray &responseData) void ColnodAPI::saveUpdatedTokens(const QByteArray &responseData)

1
src/colnod/colnodapi.h
View File

@ -27,6 +27,7 @@ class AuthToken : public serversdk::DatabaseTable {
class Timestamp : public serversdk::DatabaseTable { class Timestamp : public serversdk::DatabaseTable {
Q_OBJECT Q_OBJECT
TABLE_PROPERTY(qint64, timestamp, set_timestamp, 0) TABLE_PROPERTY(qint64, timestamp, set_timestamp, 0)
public:
QString toDateString() QString toDateString()
{ {
return QDateTime::fromMSecsSinceEpoch(a_timestamp).toString("yyyy-MM-dd hh:mm:ss.zzz"); return QDateTime::fromMSecsSinceEpoch(a_timestamp).toString("yyyy-MM-dd hh:mm:ss.zzz");

51
src/colnod/databaseapi.cpp
View File

@ -21,10 +21,10 @@ bool DatabaseAPI::open()
void DatabaseAPI::downloadData() void DatabaseAPI::downloadData()
{ {
auto updatedQuery = mDbMan->execPlainSql("SELECT id, auth_errors, description, enabled, last_update, name, no_validity, time_to_live, token_bits, token_data, token_type, subject_id, last_modified, deleted " auto updatedQuery = mDbMan->execPlainSql("SELECT * "
"FROM colnod_token WHERE EXTRACT(EPOCH FROM last_modified) * 1000 > EXTRACT(EPOCH FROM last_update) * 1000 AND deleted = false"); "FROM colnod_token WHERE last_modified > last_update AND deleted = false");
auto deletedQuery = mDbMan->execPlainSql("SELECT id " auto deletedQuery = mDbMan->execPlainSql("SELECT id "
"FROM colnod_token WHERE EXTRACT(EPOCH FROM last_modified) * 1000 > EXTRACT(EPOCH FROM last_update) * 1000 AND deleted = true"); "FROM colnod_token WHERE last_modified > last_update AND deleted = true");
emit downloadDataFinished(composeData(updatedQuery, deletedQuery)); emit downloadDataFinished(composeData(updatedQuery, deletedQuery));
} }
@ -34,24 +34,31 @@ void DatabaseAPI::processData(std::shared_ptr<Data> data)
uploadSubjects(data->updatedSubjects); uploadSubjects(data->updatedSubjects);
deleteItems(data->deletedTokens, "colnod_token"); deleteItems(data->deletedTokens, "colnod_token");
deleteItems(data->deletedSubjects, "colnod_subject"); deleteItems(data->deletedSubjects, "colnod_subject");
qInfo(database) << TAB << "-> DATA PROCESSED TO DATABASE:";
qInfo(database) << TAB << TAB << data->updatedTokens.size() << "TOKENS UPDATED";
qInfo(database) << TAB << TAB << data->deletedTokens.size() << "TOKENS DELETED";
qInfo(database) << TAB << TAB << data->updatedSubjects.size() << "SUBJECTS UPDATED";
qInfo(database) << TAB << TAB << data->deletedSubjects.size() << "SUBJECTS DELETED";
qInfo(database) << DASH;
} }
std::shared_ptr<Data> DatabaseAPI::composeData(QSqlQuery updatedQuery, QSqlQuery deletedQuery) std::shared_ptr<Data> DatabaseAPI::composeData(QSqlQuery updatedQuery, QSqlQuery deletedQuery)
{ {
auto data = std::make_shared<Data>(); auto data = std::make_shared<Data>();
while (updatedQuery.next()) { auto list = utils->tableFromQuery<Token>(updatedQuery);
auto token = std::make_shared<Token>(); for (const auto &token : list) {
token->fromSQLRecord(updatedQuery);
data->updatedTokens.insert(token->id(), token); data->updatedTokens.insert(token->id(), token);
} }
while (deletedQuery.next()) { while (deletedQuery.next()) {
data->deletedTokens.append(deletedQuery.value(0).toString()); data->deletedTokens.append(deletedQuery.value(0).toString());
} }
qDebug(database) << "DATABASE DATA:"; qInfo(database) << TAB << "-> DATABASE DATA:";
qDebug(database) << TAB << data->updatedTokens.size() << "TOKENS TO UPDATE"; qInfo(database) << TAB << TAB << data->updatedTokens.size() << "TOKENS WERE MODIFIED";
qDebug(database) << TAB << data->deletedTokens.size() << "TOKENS TO DELETE"; qInfo(database) << TAB << TAB << data->deletedTokens.size() << "TOKENS MARKED FOR DELETION";
qDebug(database) << DASH ; qInfo(database) << DASH ;
return data; return data;
@ -85,6 +92,10 @@ void DatabaseAPI::setLastUpdate(QHash<QString, std::shared_ptr<Token>> tokens)
query.bindValue(":id", token->id()); query.bindValue(":id", token->id());
mDbMan->executeQuery(query); mDbMan->executeQuery(query);
} }
qInfo(database) << EQUALS;
qInfo(database) << "Synchronize last update in database with colnod";
qInfo(database) << EQUALS;
} }
qint64 DatabaseAPI::getNow() qint64 DatabaseAPI::getNow()
@ -112,7 +123,6 @@ void DatabaseAPI::initNewTokens()
void DatabaseAPI::uploadTokens(QHash<QString, std::shared_ptr<Token>> tokens) void DatabaseAPI::uploadTokens(QHash<QString, std::shared_ptr<Token>> tokens)
{ {
// TODO:
auto query = mDbMan->createQuery(); auto query = mDbMan->createQuery();
for (const auto &token : tokens) { for (const auto &token : tokens) {
if (exists(token->id(), "colnod_token")) { if (exists(token->id(), "colnod_token")) {
@ -127,10 +137,17 @@ void DatabaseAPI::uploadTokens(QHash<QString, std::shared_ptr<Token>> tokens)
void DatabaseAPI::uploadSubjects(QHash<QString, std::shared_ptr<Subject>> subjects) void DatabaseAPI::uploadSubjects(QHash<QString, std::shared_ptr<Subject>> subjects)
{ {
// TODO:
auto query = mDbMan->createQuery(); auto query = mDbMan->createQuery();
for (const auto &subject : subjects) { for (const auto &subject : subjects) {
// TODO: attributes for (const auto &attribute : subject->getAttributes()) {
if (exists(attribute->id(), "colnod_subject_attribute")) {
query = utils->updateTable(attribute.get(), "colnod_subject_attribute");
}
else {
query = utils->insertIntoTable(attribute.get(), "colnod_subject_attribute");
}
query = mDbMan->executeQuery(query);
}
// NOTE: about actions // NOTE: about actions
if (exists(subject->id(), "colnod_subject")) { if (exists(subject->id(), "colnod_subject")) {
query = utils->updateTable(subject.get(), "colnod_subject"); query = utils->updateTable(subject.get(), "colnod_subject");
@ -144,7 +161,6 @@ void DatabaseAPI::uploadSubjects(QHash<QString, std::shared_ptr<Subject>> subjec
void DatabaseAPI::deleteItems(QList<QString> ids, const QString &tableName) void DatabaseAPI::deleteItems(QList<QString> ids, const QString &tableName)
{ {
// TODO:
auto query = mDbMan->createQuery(); auto query = mDbMan->createQuery();
QString nowString = QDateTime::fromMSecsSinceEpoch(getLastSync()).toString("yyyy-MM-dd hh:mm:ss.zzz"); QString nowString = QDateTime::fromMSecsSinceEpoch(getLastSync()).toString("yyyy-MM-dd hh:mm:ss.zzz");
for (const auto &id : ids) { for (const auto &id : ids) {
@ -155,18 +171,17 @@ void DatabaseAPI::deleteItems(QList<QString> ids, const QString &tableName)
} }
} }
//TODO: return whole token std::shared_ptr<Token> DatabaseAPI::getLastUpdate(const QString &id)
QDateTime DatabaseAPI::getLastUpdate(const QString &id)
{ {
auto query = mDbMan->createQuery(); auto query = mDbMan->createQuery();
query.prepare("SELECT last_update FROM colnod_token WHERE id = :id"); query.prepare("SELECT last_update FROM colnod_token WHERE id = :id");
query.bindValue(":id", id); query.bindValue(":id", id);
mDbMan->executeQuery(query); mDbMan->executeQuery(query);
if (query.next()) { if (query.next()) {
return query.value(0).toDateTime(); return utils->tableFromQuery<Token>(query).at(0);
} }
else { else {
return QDateTime(); return nullptr;
} }
} }

2
src/colnod/databaseapi.h
View File

@ -33,7 +33,7 @@ public:
void uploadSubjects(QHash<QString, std::shared_ptr<Subject> > subjects); void uploadSubjects(QHash<QString, std::shared_ptr<Subject> > subjects);
void deleteItems(QList<QString> ids, const QString &tableName); void deleteItems(QList<QString> ids, const QString &tableName);
QDateTime getLastUpdate(const QString &id); std::shared_ptr<Token> getLastUpdate(const QString &id);
private: private:
bool exists(const QString &id, const QString &tableName); bool exists(const QString &id, const QString &tableName);

6
src/colnod/subject.cpp
View File

@ -14,6 +14,7 @@ bool Subject::fromJson(const QByteArray &data)
for (auto key : attr.keys()) { for (auto key : attr.keys()) {
auto newAtr = std::make_shared<SubjectAttribute>(); auto newAtr = std::make_shared<SubjectAttribute>();
newAtr->setId(id + " " + key);
newAtr->setSubjectId(id); newAtr->setSubjectId(id);
newAtr->setAttributeKey(key); newAtr->setAttributeKey(key);
newAtr->setAttributeValue(attr.value(key).toString()); newAtr->setAttributeValue(attr.value(key).toString());
@ -25,3 +26,8 @@ bool Subject::fromJson(const QByteArray &data)
bool res = fromVariantMap(variant); bool res = fromVariantMap(variant);
return res; return res;
} }
QList<std::shared_ptr<SubjectAttribute> > Subject::getAttributes()
{
return attributes;
}

2
src/colnod/subject.h
View File

@ -30,6 +30,8 @@ public:
explicit Subject(const QString& name); explicit Subject(const QString& name);
bool fromJson(const QByteArray &data) override; bool fromJson(const QByteArray &data) override;
QList<std::shared_ptr <SubjectAttribute>> getAttributes();
private: private:
QList<std::shared_ptr <SubjectAttribute>> attributes; QList<std::shared_ptr <SubjectAttribute>> attributes;

79
src/colnod/syncmanager.cpp
View File

@ -25,17 +25,28 @@ bool SyncManager::init()
void SyncManager::run() void SyncManager::run()
{ {
qint64 syncBegin = database->getNow(); qint64 syncBegin = database->getNow();
qInfo(syncmanager) << EQUALS;
qInfo(syncmanager) << "SYNCHRONIZATION BEGIN";
database->initNewTokens(); database->initNewTokens();
downloadData(); downloadData();
connect(this, &SyncManager::dataReady, [=](){ connect(this, &SyncManager::dataReady, [=](){
qDebug(syncmanager) << "FINISH";
prepareData(); prepareData();
connect(this, &SyncManager::dataProcessFinished, [=](){ connect(this, &SyncManager::dataProcessFinished, [=](){
updateLastUpdate(syncBegin); updateLastUpdate(syncBegin);
connect(this, &SyncManager::syncFinished, [=](){ connect(this, &SyncManager::syncFinished, [=](){
database->setLastSync(syncBegin); database->setLastSync(syncBegin);
qInfo(syncmanager) << "Update last synchronization time to time of this sync ("
<< QDateTime::fromMSecsSinceEpoch(syncBegin).toString("yyyy-MM-dd hh:mm:ss.zzz")
<< ")";
qInfo(syncmanager) << EQUALS;
qInfo(syncmanager) << "SYNCHRONIZATION FINISHED";
qInfo(syncmanager) << EQUALS;
QCoreApplication::exit(); QCoreApplication::exit();
}); });
}); });
@ -46,9 +57,11 @@ void SyncManager::run()
void SyncManager::downloadData() void SyncManager::downloadData()
{ {
qInfo(syncmanager) << EQUALS << endl; qInfo(syncmanager) << EQUALS;
qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN" << endl; qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN";
qInfo(syncmanager) << UNDERSCORE; qInfo(syncmanager) << DASH;
qInfo(syncmanager) << "Modified data since last synchronization (" << mlastSync.toDateString() << "):";
qInfo(syncmanager) << DASH;
connect(colnod.get(), &ColnodAPI::downloadDataFinished, this, &SyncManager::saveColnodData); connect(colnod.get(), &ColnodAPI::downloadDataFinished, this, &SyncManager::saveColnodData);
connect(database.get(), &DatabaseAPI::downloadDataFinished, this, &SyncManager::saveDatabaseData); connect(database.get(), &DatabaseAPI::downloadDataFinished, this, &SyncManager::saveDatabaseData);
@ -59,44 +72,74 @@ void SyncManager::downloadData()
// finds conflicts -> changes on both sides (colnod and DB), apply newer change // finds conflicts -> changes on both sides (colnod and DB), apply newer change
void SyncManager::prepareData() void SyncManager::prepareData()
{ {
qInfo(syncmanager) << EQUALS;
qInfo(syncmanager) << "PREPARING DATA -> BEGIN";
qInfo(syncmanager) << DASH;
qInfo(syncmanager) << "Resolve conflicts ( changes on same item on both Colnod and DB ):";
qInfo(syncmanager) << DASH;
colnodDataToProcess = std::make_shared<Data>(*colnodData); colnodDataToProcess = std::make_shared<Data>(*colnodData);
databaseDataToProcess = std::make_shared<Data>(*databaseData); databaseDataToProcess = std::make_shared<Data>(*databaseData);
QList<QString> idsToDelete; QList<QString> conflicts;
QList<QString> alreadySynced;
for (const auto &colnodToken : colnodDataToProcess->updatedTokens) { for (const auto &colnodToken : colnodDataToProcess->updatedTokens) {
if (databaseDataToProcess->updatedTokens.contains(colnodToken->id())) { if (databaseDataToProcess->updatedTokens.contains(colnodToken->id())) {
if (databaseDataToProcess->updatedTokens.value(colnodToken->id()) > colnodToken) { if (databaseDataToProcess->updatedTokens.value(colnodToken->id()) > colnodToken) {
idsToDelete.append(colnodToken->id()); conflicts.append(colnodToken->id());
} }
continue; continue;
} }
auto dbLastUpdate = database->getLastUpdate(colnodToken->id()); auto token = database->getLastUpdate(colnodToken->id());
if ( !dbLastUpdate.isNull() ) if ( !token )
// TODO: compare whole structure of token if (token == colnodToken)
if (dbLastUpdate == colnodToken->lastUpdate()) alreadySynced.append(colnodToken->id());
idsToDelete.append(colnodToken->id());
} }
for (const auto &id : idsToDelete) for (const auto &id : conflicts)
colnodDataToProcess->updatedTokens.remove(id);
for (const auto &id : alreadySynced)
colnodDataToProcess->updatedTokens.remove(id); colnodDataToProcess->updatedTokens.remove(id);
idsToDelete.clear(); qInfo(syncmanager) << TAB << "-> COLNOD DATA - CHANGES:";
qInfo(syncmanager) << TAB << TAB << conflicts.size() << "TOKENS WERE MODIFIED IN DATABASE";
qInfo(syncmanager) << TAB << TAB << alreadySynced.size() << "TOKENS WERE ALREADY SYNCED";
qInfo(syncmanager) << DASH;
conflicts.clear();
for (const auto &databaseToken : databaseData->updatedTokens) { for (const auto &databaseToken : databaseData->updatedTokens) {
if (colnodDataToProcess->updatedTokens.contains(databaseToken->id())) { if (colnodDataToProcess->updatedTokens.contains(databaseToken->id())) {
if (databaseToken > colnodDataToProcess->updatedTokens.value(databaseToken->id())) if (databaseToken > colnodDataToProcess->updatedTokens.value(databaseToken->id()))
idsToDelete.append(databaseToken->id()); conflicts.append(databaseToken->id());
} }
} }
for (const auto &id : idsToDelete) for (const auto &id : conflicts)
colnodDataToProcess->updatedTokens.remove(id); colnodDataToProcess->updatedTokens.remove(id);
qInfo(syncmanager) << TAB << "-> DATABASE DATA - CHANGES:";
qInfo(syncmanager) << TAB << TAB << conflicts.size() << "TOKENS WERE MODIFIED IN COLNOD";
qInfo(syncmanager) << DASH;
qInfo(syncmanager) << "PREPARING DATA -> FINISHED";
qInfo(syncmanager) << EQUALS;
} }
void SyncManager::processData() void SyncManager::processData()
{ {
qInfo(syncmanager) << EQUALS;
qInfo(syncmanager) << "PROCESSING DATA -> BEGIN";
qInfo(syncmanager) << DASH;
qInfo(syncmanager) << "Push changes to both Colnod ad Database:";
qInfo(syncmanager) << DASH;
database->processData(colnodDataToProcess); database->processData(colnodDataToProcess);
connect(colnod.get(), &ColnodAPI::dataProcessFinished, this, &SyncManager::dataProcessFinished); connect(colnod.get(), &ColnodAPI::dataProcessFinished, this, &SyncManager::dataProcessFinished);
colnod->processData(databaseDataToProcess); colnod->processData(databaseDataToProcess);
qInfo(syncmanager) << "PROCESSING DATA -> FINISHED";
qInfo(syncmanager) << EQUALS;
} }
void SyncManager::updateLastUpdate(qint64 syncBegin) void SyncManager::updateLastUpdate(qint64 syncBegin)
@ -112,13 +155,13 @@ void SyncManager::updateLastUpdate(qint64 syncBegin)
void SyncManager::areDataReady() void SyncManager::areDataReady()
{ {
if (++dataAmount == 2) if (++dataAmount == 2) {
qDebug(syncmanager) << "DOWNLOADING DATA -> FINISHED" << endl << endl; qDebug(syncmanager) << "DOWNLOADING DATA -> FINISHED";
qDebug(syncmanager) << EQUALS; qDebug(syncmanager) << EQUALS;
qDebug(syncmanager) << EQUALS << endl << endl;
emit dataReady(); emit dataReady();
}
} }
void SyncManager::saveColnodData(const std::shared_ptr<Data> &data) void SyncManager::saveColnodData(const std::shared_ptr<Data> &data)

2
src/colnod/token.cpp
View File

@ -93,7 +93,7 @@ void Token::fromSQLRecord(const QSqlQuery& query)
setNoValidity(query.value(6).toBool()); setNoValidity(query.value(6).toBool());
setEnabled(query.value(3).toBool()); setEnabled(query.value(3).toBool());
QDateTime date = query.value(4).toDateTime(); QDateTime date = query.value(4).toDateTime();
setLastUpdate(date); // WARNING: resolve how lastUpdate should me stored setLastUpdate(date);
setName(query.value(5).toString()); setName(query.value(5).toString());
a_timeToLive = query.value(7).toInt(); a_timeToLive = query.value(7).toInt();
a_tokenBits = query.value(8).toInt(); a_tokenBits = query.value(8).toInt();