ondra/colnod-connector
1
0
Fork 1
mirror of https://gitlab.com/spaceti-app/integrations/acs/colnod-connector.git synced 2026-07-12 17:00:39 +02:00
Code Issues Projects Releases Wiki Activity

add 5 min loop

Browse Source
This commit is contained in:
ondra 2020-12-22 16:05:21 +01:00
parent de021e0834
commit be9064c46e
10 changed files with 279 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/app/main.cpp
View File

@ -8,6 +8,7 @@
#include "../colnod/colnodapi.h"
#include "../colnod/syncmanager.h"
#include "../colnod/logutils.h"
int main(int argc, char *argv[])
{
@ -29,7 +30,9 @@ int main(int argc, char *argv[])
// Logger initializiaton
serversdk::initAppLogger();
// serversdk::initAppLogger();
LogUtils::initLogger(false);
// Log messge Filtering
QStringList filters;
@ -47,11 +50,14 @@ int main(int argc, char *argv[])
return -1;
}
syncManager.init();
QCoreApplication::connect(&syncManager, &SyncManager::end, [&]{
QTimer::singleShot(300000, &syncManager, &SyncManager::startSynchronization);
});
// Synchronizaiton
if (!syncManager.startSynchronization()) {
qCritical() << "Not possible to start synchronization";
return -1;
}
syncManager.startSynchronization();
return QCoreApplication::exec();
}

6
src/colnod/colnod.pri
View File

@ -9,7 +9,8 @@ SOURCES += \
$$PWD/databaseapi.cpp \
$$PWD/subject.cpp \
$$PWD/syncmanager.cpp \
$$PWD/token.cpp
$$PWD/token.cpp \
$$PWD/logutils.cpp
HEADERS += \
$$PWD/colnodapi.h \
@ -17,6 +18,7 @@ HEADERS += \
$$PWD/databaseapi.h \
$$PWD/subject.h \
$$PWD/syncmanager.h \
$$PWD/token.h
$$PWD/token.h \
$$PWD/logutils.h
include(../../server-sdk/serversdk.pri)

5
src/colnod/colnodapi.cpp
View File

@ -155,8 +155,9 @@ bool ColnodAPI::downloadData(const qint64 &time)
return true;
}
void ColnodAPI::processData(const std::shared_ptr<Data> &data)
void ColnodAPI::processData(const std::shared_ptr<PreparedData> &prepdata)
{
auto data = prepdata->databaseData;
// emit signal when finished
auto checkData = [=]() {
int dataCount = data->incrementCounter();
@ -185,7 +186,7 @@ void ColnodAPI::processData(const std::shared_ptr<Data> &data)
!data->updatedSubjects.isEmpty() || !data->deletedSubjects.isEmpty()) {
qInfo(colnod) << dash;
}
emit dataProcessFinished(mValid);
emit dataProcessFinished(mValid, prepdata);
}
};

4
src/colnod/colnodapi.h
View File

@ -72,7 +72,7 @@ public:
* @brief Push all data prepare during synchronization to Colnod
* @param data Data which are pushing to Colnod
*/
void processData(const std::shared_ptr<Data> &data);
void processData(const std::shared_ptr<PreparedData> &prepdata);
/**
* @brief Endpoint of Colnod API
@ -124,7 +124,7 @@ signals:
* @brief Signal which transfer data to Sync Manager
*/
void downloadDataFinished(std::shared_ptr<Data> = std::shared_ptr<Data>());
void dataProcessFinished(bool success = true);
void dataProcessFinished(bool success = true, std::shared_ptr<PreparedData> = nullptr);
void authTokenUpdated(bool success);
private:

4
src/colnod/databaseapi.cpp
View File

@ -326,3 +326,7 @@ QString DatabaseAPI::getSubjectNameByTokenId(const QString &tokenId)
qWarning() << "Token:" << tokenId << "is not assigned to any subject";
return "(No subject)";
}
bool DatabaseAPI::isOpen() {
return mDbMan->isOpen();
}

1
src/colnod/databaseapi.h
View File

@ -106,6 +106,7 @@ public:
* @brief Check if this sync was without any problem
*/
bool isValid();
bool isOpen();
// Helper functions for naming new tokens
qint16 getNumOfTokensOfSubject(const QString &id);

97
src/colnod/logutils.cpp Normal file
View File

@ -0,0 +1,97 @@
#include "logutils.h"
namespace LogUtils{
static bool verbose;
void initLogger(bool vb)
{
verbose = vb;
if (!QDir(logFolderName).exists()) {
QDir().mkdir(logFolderName);
}
deleteOldLogs();
initLogFileName();
QFile outFile(logFileName);
if (outFile.open(QIODevice::WriteOnly | QIODevice::Append))
{
qInstallMessageHandler(myMessageHandler);
} else {
qWarning() << "Cannot open log file.";
}
}
void myMessageHandler(QtMsgType type, const QMessageLogContext &context, const QString &msg)
{
QFile outFileCheck(logFileName);
QFileInfo fileInfo(outFileCheck);
if (fileInfo.created().date() < QDateTime::currentDateTime().date()) {
QFile logFile(logFileName);
logFile.close();
deleteOldLogs();
initLogFileName();
}
QByteArray localMsg = msg.toLocal8Bit();
QString fileString(context.file);
fileString = fileString.mid(fileString.lastIndexOf("\\")+1);
auto date = QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz");
QString txt;
switch (type) {
case QtDebugMsg:
if (verbose)
txt = date + " | " + QString("DEBUG").leftJustified(8,' ') + " | " + localMsg;
break;
case QtInfoMsg:
txt = date + " | " + QString("INFO").leftJustified(8,' ') + " | " + localMsg;
break;
case QtWarningMsg:
txt = date + " | " + QString("WARNING").leftJustified(8,' ') + " | " + QString(fileString + ':' + QString::number(context.line)).rightJustified(19, ' ') + " | " + localMsg;
break;
case QtCriticalMsg:
txt = date + " | " + QString("CRITICAL").leftJustified(8,' ') + " | " + QString(fileString + ':' + QString::number(context.line)).rightJustified(19, ' ') + " | " + localMsg;
break;
case QtFatalMsg:
txt = date + " | " + QString("FATAL").leftJustified(8,' ') + " | " + QString(fileString + ':' + QString::number(context.line)).rightJustified(19, ' ') + " | " + localMsg;
abort();
}
if (!txt.isEmpty()) {
QFile outFile(logFileName);
outFile.open(QIODevice::WriteOnly | QIODevice::Append);
QTextStream textStream(&outFile);
if (!txt.isEmpty()) textStream << txt << endl;
}
}
void deleteOldLogs(int days)
{
auto today = QDateTime::currentDateTime();
QDir dir;
dir.setFilter(QDir::Files);
dir.setSorting(QDir::Time | QDir::Reversed);
dir.setPath(logFolderName);
auto filesInfo = dir.entryInfoList();
int i = 0;
if (!filesInfo.isEmpty()) {
while (filesInfo[i].created().daysTo(today) > days) {
QString path = filesInfo[i++].absoluteFilePath();
QFile file(path);
file.remove();
qDebug() << "File:" << path << "has been deleted!";
}
}
}
void initLogFileName()
{
logFileName = QString(logFolderName + "/log_%1.txt").arg(QDate::currentDate().toString("yyyy_MM_dd"));
}
void setLogFolder(QString path)
{
logFolderName = path;
}
}

25
src/colnod/logutils.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#include <QByteArray>
#include <QString>
#include <QDateTime>
#include <QDate>
#include <QDebug>
#include <QDir>
namespace LogUtils
{
void initLogger(bool vb = false);
void myMessageHandler(QtMsgType type, const QMessageLogContext &context, const QString &msg);
void deleteOldLogs(int days = 90);
void initLogFileName();
void setLogFolder(QString path);
static QString logFileName;
static QString logFolderName = "/var/spaceti/log/colnod";
//static constexpr int logSize = 1024 * 0;
//static constexpr int logFileCount = 0;
};

140
src/colnod/syncmanager.cpp
View File

@ -20,64 +20,54 @@ bool SyncManager::databaseInitialization()
return false;
}
// Get last synchronizaiton date
qint64 lastsync_in_ms = database->getLastSync();
if (lastsync_in_ms == -1) {
qCritical(syncmanager) << "Not possible to get last synchronization date";
return false;
}
// here -> nothing can fail
mlastSync.setTimestamp(lastsync_in_ms);
colnod = std::make_shared<ColnodAPI>(&reqMan, mDryMode);
return true;
}
bool SyncManager::startSynchronization()
{
// Need current time from db server
qint64 syncBegin = database->getNow();
if (syncBegin == -1) {
qCritical(syncmanager) << "Not possible to get last sync date";
return false;
}
qInfo(syncmanager) << tab << "- begin sync ( previous sync" << mlastSync.toDateString() << ")";
void SyncManager::init() {
// GETTING DATA FROM DATABASE
auto databaseData = database->downloadData();
if (!databaseData) {
qCritical(syncmanager) << "not possible to get data from database";
return false;
}
// NOW I HAVE DATA FROM DATABASE
colnod->downloadData(mlastSync.timestamp());
// qint64 begin = syncBegin;
connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](const std::shared_ptr<Data> &colnodData) {
// NOW I HAVE DATA FROM COLNOD
if (!colnodData) {
qCritical(syncmanager) << "Not possible to get correct data from colnod";
QCoreApplication::exit(-1);
emit end();
return;
}
std::unique_ptr<PreparedData> preparedData = prepareData(databaseData, colnodData);
// GETTING DATA FROM DATABASE
auto databaseData = database->downloadData();
if (!databaseData) {
emit end();
return;
}
// NOW I HAVE DATA FROM DATABASE
std::shared_ptr<PreparedData> preparedData = prepareData(databaseData, colnodData);
if (!preparedData) {
qCritical(syncmanager) << "Not possible to compose data";
QCoreApplication::exit(-1);
emit end();
return;
}
if (!database->processData(preparedData->colnodData)) {
qCritical(syncmanager) << "Not possible to upload data to database";
err = true;
return;
}
colnod->processData(preparedData);
// data is processed to other sides
connect(colnod.get(), &ColnodAPI::dataProcessFinished, [&,syncBegin](bool success){
});
connect(colnod.get(), &ColnodAPI::dataProcessFinished, [=](bool success, std::shared_ptr<PreparedData> preparedData){
if (!success) {
qCritical(syncmanager) << "Not possible to upload data to colnod";
err = true;
}
// REUPDATE tokens pushed from database to colnod (cause of lastUpdate time)
@ -85,57 +75,107 @@ bool SyncManager::startSynchronization()
// download data changes in this sync
colnod->updateAuthToken();
connect(colnod.get(), &ColnodAPI::authTokenUpdated, [&,syncBegin](bool success) {
QTimer timer;
timer.setSingleShot(true);
QEventLoop loop;
QMetaObject::Connection myConnect = connect(colnod.get(), &ColnodAPI::authTokenUpdated, [this, &loop](bool success){
if (!success) {
qCritical(syncmanager) << "Cannot login to Colnod.";
emit end();
}
qDebug() << "test loop";
loop.quit();
});
// connect(colnod.get(), &ColnodAPI::authTokenUpdated, [this](bool success) {\
// if (!success) {
// qCritical(syncmanager) << "Cannot login to Colnod.";
// emit end();
// }
// qDebug() << "test loop";
// return;
// });
// connect( colnod.get(), &ColnodAPI::authTokenUpdated, &loop, &QEventLoop::quit);
connect( &timer, &QTimer::timeout, &loop, &QEventLoop::quit );
timer.start(30000);
loop.exec();
loop.deleteLater();
disconnect(myConnect);
qDebug() << "Loop ended";
if(!timer.isActive()) {
qCritical("Cannot login to Colnod");
emit end();
return;
}
Timestamp time;
time.setTimestamp(syncBegin - 75 * 1000);
time.setTimestamp(syncBegin - 100 * 1000);
auto tokensRequest = colnod->getTokensUpdated(time);
// updated data are ready
connect(tokensRequest.get(), &serversdk::Request::finished, [=](){
connect(tokensRequest.get(), &serversdk::Request::finished, [=](){
auto uploadedData = std::make_shared<Data>();
uploadedData->loadTokensFromJson(tokensRequest->responseData());
if (!uploadedData) {
qCritical(syncmanager) << "Not possible to get correct data from colnod";
QCoreApplication::exit(-1);
emit end();
return;
}
if (!database->uploadTokens(uploadedData->updatedTokens, {"lastModified"})) {
qCritical(syncmanager) << "Not possible to upload data to database";
QCoreApplication::exit(-1);
emit end();
return;
}
database->reupdateDeleted(databaseData->deletedTokens);
database->reupdateDeleted(preparedData->databaseData->deletedTokens);
if (!database->isValid() || !colnod->isValid()) {
if (!database->isValid() || !colnod->isValid() || err || !database->setLastSync(syncBegin)) {
qWarning(syncmanager) << "Unsuccessful FINISH, check warnings";
}
else {
// set new time of last sync to this time of this run
if (!database->setLastSync(syncBegin)) {
qWarning(syncmanager) << "Unsuccessful FINISH, check warnings";
}
} else {
qInfo(syncmanager) << "Successful FINISH";
}
QCoreApplication::exit();
});
emit end();
});
});
}
bool SyncManager::startSynchronization()
{
if (!database->isOpen() ) {
if (!database->open()) {
qCritical(syncmanager) << "Cannot open database.";
return false;
}
}
qint64 lastsync_in_ms = database->getLastSync();
if (lastsync_in_ms == -1) {
qCritical(syncmanager) << "Not possible to get last synchronization date";
return false;
}
mlastSync.setTimestamp(lastsync_in_ms);
// Need current time from db server
err = false;
syncBegin = database->getNow();
if (syncBegin == -1) {
qCritical(syncmanager) << "Not possible to get last sync date";
emit end();
return false;
}
qint64 offset = 75 * 1000;
qInfo(syncmanager) << tab << "Begin sync ( previous sync" << mlastSync.toDateString() << ")";
colnod->downloadData(mlastSync.timestamp() - offset);
});
colnod->processData(preparedData->databaseData);
});
return true;
}
// goes through downloaded data and check if consist conflicts (both sides edit), already sync/deleted tokens or new tokens
std::unique_ptr<PreparedData> SyncManager::prepareData(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData)
std::shared_ptr<PreparedData> SyncManager::prepareData(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData)
{
auto preparedColnodData = std::make_shared<Data>();
auto preparedDatabaseData = std::make_shared<Data>();

10
src/colnod/syncmanager.h
View File

@ -13,6 +13,7 @@ public:
explicit SyncManager(bool dryMode);
bool databaseInitialization();
void init();
bool startSynchronization();
private:
@ -22,7 +23,12 @@ private:
* @param data from colnod
* @return prepared data to be pushed into colnod and db
*/
std::unique_ptr<PreparedData> prepareData(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData);
std::shared_ptr<PreparedData> prepareData(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData);
signals:
void error();
void end();
private:
/**
@ -37,6 +43,8 @@ private:
serversdk::RequestManager reqMan;
serversdk::DatabaseManager dbMan;
Timestamp mlastSync;
qint64 syncBegin;
bool err;
bool mDryMode;
};