michalks/colnod-connector
1
0
Fork 0
forked from ondra/colnod-connector
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'suggest/process-data' into suggest/preapre-data-use-old

Browse Source 
# Conflicts:
#	src/colnod/databaseapi.cpp
#	src/colnod/databaseapi.h
#	src/colnod/syncmanager.cpp
#	src/colnod/syncmanager.h
#	test/auto/colnod/tst_colnodapi.cpp
#	test/auto/syncmanager/tst_syncmanager.cpp
This commit is contained in:
Filip Bucek 2019-12-02 15:44:14 +01:00
commit 7e4a7bf074
7 changed files with 91 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/colnod/colnodapi.cpp
View File

@ -20,7 +20,7 @@ ColnodAPI::ColnodAPI(serversdk::RequestManager* reqMan, const bool &dryMode, QSt
* @return signal with nullptr when something went wrong
* @return signal with data when valid data returned
*/
void ColnodAPI::downloadData(const Timestamp &time)
void ColnodAPI::downloadData(const qint64 &time)
{
// Have to get Login
auto login = requestLogin();
@ -30,7 +30,6 @@ void ColnodAPI::downloadData(const Timestamp &time)
emit downloadDataFinished(nullptr);
return;
}
AuthToken token;
auto responseData = login->responseData();
if (responseData.isEmpty()) {
@ -53,7 +52,7 @@ void ColnodAPI::downloadData(const Timestamp &time)
if (!checkRequest(request)) {
qWarning(colnod) << "ERROR during calling" << request->url() << ", request unsuccessful";
mValid = false;
emit downloadDataFinished(nullptr);
emit downloadDataFinished(nullptr); // TODO: think about it, will the other if passed?
}
else if (!checkResponse(request->responseData())) {
qWarning(colnod) << "Data from" << request->url() << "are invalid";
@ -61,31 +60,35 @@ void ColnodAPI::downloadData(const Timestamp &time)
}
};
// if I have all data emit downloadDataFinished
connect(this, &ColnodAPI::partOfDataReady, this, &ColnodAPI::areDataReady);
auto tokensRequest = getTokensUpdated(time);
Timestamp timestamp;
timestamp.setTimestamp(time);
auto tokensRequest = getTokensUpdated(timestamp);
connect(tokensRequest.get(), &serversdk::Request::finished, errorHandler);
connect(tokensRequest.get(), &serversdk::Request::finished, [=](){
saveUpdatedTokens(data, tokensRequest->responseData());
emit partOfDataReady(data);
});
auto deletedTokensRequest = getTokensDeleted(time);
auto deletedTokensRequest = getTokensDeleted(timestamp);
connect(deletedTokensRequest.get(), &serversdk::Request::finished, errorHandler);
connect(deletedTokensRequest.get(), &serversdk::Request::finished, [=](){
saveDeletedTokens(data, deletedTokensRequest->responseData());
emit partOfDataReady(data);
});
auto subjectsRequest = getSubjectsUpdated(time);
auto subjectsRequest = getSubjectsUpdated(timestamp);
connect(subjectsRequest.get(), &serversdk::Request::finished, errorHandler);
connect(subjectsRequest.get(), &serversdk::Request::finished, [=](){
saveUpdatedSubjects(data, subjectsRequest->responseData());
emit partOfDataReady(data);
});
auto deletedSubjectsRequest = getSubjectsDeleted(time);
auto deletedSubjectsRequest = getSubjectsDeleted(timestamp);
connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, errorHandler);
connect(deletedSubjectsRequest.get(), &serversdk::Request::finished, [=](){
saveDeletedSubjects(data, deletedSubjectsRequest->responseData());
@ -128,14 +131,14 @@ std::shared_ptr<serversdk::Request> ColnodAPI::requestLogin()
{
Q_ASSERT(mReqMan);
LoginResp login;
login.set_username(userName());
login.setUsername(userName());
// password
QString encodedPassword = QCryptographicHash::hash(
password().toLocal8Bit(), // -> bytearray
QCryptographicHash::Sha512 // SHA 512
).toHex().toUpper();
login.set_password(encodedPassword);
login.setPassword(encodedPassword);
auto loginRequest = composeRequest("/AaaManager/authSession", login.jsonData());
loginRequest->setUserName(login.username());
@ -254,7 +257,7 @@ qint64 ColnodAPI::lastSync() const
void ColnodAPI::setlastSync(const qint64 &lastSync)
{
mLastSync.set_timestamp(lastSync);
mLastSync.setTimestamp(lastSync);
}

23
src/colnod/colnodapi.h
View File

@ -10,31 +10,24 @@ Q_DECLARE_LOGGING_CATEGORY(colnod)
class LoginResp : public serversdk::DatabaseTable {
Q_OBJECT
TABLE_PROPERTY(QString, username, set_username, "")
TABLE_PROPERTY(QString, password, set_password, "")
TABLE_PROPERTY(QString, username, setUsername, "")
TABLE_PROPERTY(QString, password, setPassword, "")
};
class AuthToken : public serversdk::DatabaseTable {
Q_OBJECT
TABLE_PROPERTY(QString, id, set_id, QString())
TABLE_PROPERTY(QString, id, setId, QString())
};
class Timestamp : public serversdk::DatabaseTable {
Q_OBJECT
TABLE_PROPERTY(qint64, timestamp, set_timestamp, 0)
TABLE_PROPERTY(qint64, timestamp, setTimestamp, 0)
public:
Timestamp() = default;
explicit Timestamp(const Timestamp &time)
void setTimestamp(QDateTime time)
{
this->set_timestamp(time.timestamp());
}
explicit Timestamp(qint64 time)
{
set_timestamp(time);
}
explicit Timestamp(QDateTime time)
{
set_timestamp(time.toMSecsSinceEpoch());
setTimestamp(time.toMSecsSinceEpoch());
}
QString toDateString()
@ -50,7 +43,7 @@ public:
ColnodAPI() = default;
ColnodAPI(serversdk::RequestManager* reqManconst, const bool &dryMode = false, QString userName = "spaceti", QString password = "spaceti1");
void downloadData(const Timestamp &time);
void downloadData(const qint64 &time);
void processData(const std::shared_ptr<Data> &data);
// New approach

14
src/colnod/databaseapi.cpp
View File

@ -118,7 +118,7 @@ void DatabaseAPI::setLastSync(qint64 timestamp)
}
}
void DatabaseAPI::setLastUpdate(const QHash<QString, std::shared_ptr<Token>> &tokens)
void DatabaseAPI::reupdateUpdated(const QHash<QString, std::shared_ptr<Token>> &tokens)
{
auto query = mDbMan->createQuery();
for (const auto &token : tokens){
@ -135,7 +135,8 @@ void DatabaseAPI::setLastUpdate(const QHash<QString, std::shared_ptr<Token>> &to
qInfo(database) << equals;
}
void DatabaseAPI::setLastUpdate(const QList<QString>& ids)
void DatabaseAPI::reupdateDeleted(const QList<QString>& ids)
{
auto query = mDbMan->createQuery();
for (const auto &id : ids){
@ -148,7 +149,6 @@ void DatabaseAPI::setLastUpdate(const QList<QString>& ids)
}
}
}
}
qint64 DatabaseAPI::getNow()
@ -193,14 +193,6 @@ bool DatabaseAPI::uploadTokens(const QHash<QString, std::shared_ptr<Token>> &tok
for (const auto &token : tokens) {
// Get token
QSqlQuery query;
if (exists(token->id(), "colnod_token")) {
query = utils->updateTable(token.get(), "colnod_token");
qInfo(database).noquote() << Message::massageTemplate("Updating", "token", "database", token->name());

4
src/colnod/databaseapi.h
View File

@ -31,8 +31,8 @@ public:
bool processData(const std::shared_ptr<Data> &data);
qint64 getLastSync();
void setLastSync(qint64 timestamp);
void setLastUpdate(const QHash<QString, std::shared_ptr<Token> > &tokens);
void setLastUpdate(const QList<QString>& ids);
void reupdateUpdated(const QHash<QString, std::shared_ptr<Token> > &tokens);
void reupdateDeleted(const QList<QString>& ids);
qint64 getNow();
void initNewTokens();

149
src/colnod/syncmanager.cpp
View File

@ -13,9 +13,6 @@ SyncManager::SyncManager(bool dryMode)
bool SyncManager::databaseInitialization()
{
database = std::make_shared<DatabaseAPI>(&dbMan, mDryMode);
// connect(database.get(), &DatabaseAPI::error, [=](){
// QCoreApplication::exit(1);
// });
// Open database
if (!database->open()) {
@ -31,7 +28,7 @@ bool SyncManager::databaseInitialization()
}
// here -> nothing can fail
mlastSync.set_timestamp(database->getLastSync());
mlastSync.setTimestamp(database->getLastSync());
colnod = std::make_shared<ColnodAPI>(&reqMan, mDryMode);
return true;
}
@ -49,9 +46,11 @@ bool SyncManager::startSynchronization()
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "SYNCHRONIZATION BEGIN";
// TODO: solve this in prepare data
database->initNewTokens();
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN";
qInfo(syncmanager) << dash;
qInfo(syncmanager).noquote() << "Modified data since last synchronization (" << mlastSync.toDateString() << "):";
qInfo(syncmanager) << dash;
// GETTING DATA FROM DATABASE
auto databaseData = database->downloadData();
@ -61,9 +60,11 @@ bool SyncManager::startSynchronization()
}
// NOW I HAVE DATA FROM DATABASE
colnod->downloadData(syncBegin);
connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](std::shared_ptr<Data> colnodData) {
qInfo(syncmanager) << "DOWNLOADING DATA -> FINISHED";
qInfo(syncmanager) << equals;
colnod->downloadData(Timestamp(syncBegin));
connect(colnod.get(), &ColnodAPI::downloadDataFinished, [=](const std::shared_ptr<Data>& colnodData) {
if (!colnodData) {
qCritical(syncmanager) << "Not possible to get correct data from colnod";
QCoreApplication::exit(-1);
@ -71,9 +72,7 @@ bool SyncManager::startSynchronization()
}
// NOW I SHOULD HAVE VALID DATA FROM COLNOD
// I can process data
qDebug(syncmanager) << "I have colnod and database data";
qDebug(syncmanager) << "updated token count" << colnodData->updatedTokens.count();
std::unique_ptr<PreparedData> preparedData = prepareDataNew(databaseData, colnodData);
if (!preparedData) {
@ -82,46 +81,66 @@ bool SyncManager::startSynchronization()
return;
}
// NOT FINISHED scolnod->processData(preparedData->colnodData);
});
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "PROCESSING DATA -> BEGIN";
qInfo(syncmanager) << dash;
qInfo(syncmanager) << "Push changes to both Colnod and Database:";
qInfo(syncmanager) << dash;
return true;
//downloadData();
//connect(this, &SyncManager::dataReady, processDownloadedData);
connect(this, &SyncManager::dataReady, [=](){
//prepareData();
connect(this, &SyncManager::dataProcessFinished, [=](){
if (!database->processData(preparedData->databaseData)) {
qCritical(syncmanager) << "Not possible to upload data to database";
}
// data is processed to other sides
connect(colnod.get(), &ColnodAPI::dataProcessFinished, [&](){
qInfo(syncmanager) << "PROCESSING DATA -> FINISHED";
qInfo(syncmanager) << equals;
updateLastUpdate(syncBegin); // TODO: change to update all attrinutes
connect(this, &SyncManager::syncFinished, [=](){
if (database->isValid() && colnod->isValid()) {
database->setLastSync(syncBegin);
// REUPDATE tokens pushed from database to colnod (cause of lastUpdate time)
Timestamp time;
time.setTimestamp(syncBegin);
database->reupdateDeleted(preparedData->databaseData->deletedTokens);
// download data changes in this sync
colnod->getTokensUpdated(time);
// updated data are ready
connect(colnod.get(), &ColnodAPI::partOfDataReady, [=](const std::shared_ptr<Data> &uploadedData){
if (!uploadedData) {
qCritical(syncmanager) << "Not possible to get correct data from colnod";
QCoreApplication::exit(-1);
return;
}
else {
if (!database->uploadTokens(uploadedData->updatedTokens)) {
qCritical(syncmanager) << "Not possible to upload data to database";
}
database->reupdateDeleted(uploadedData->deletedTokens);
if (!database->isValid() || !colnod->isValid()) {
qWarning(syncmanager) << "Synchronization was unsuccessful, next run of sync will be with same last sync time, check warnings";
}
else {
database->setLastSync(syncBegin);
qInfo(syncmanager).noquote() << "Update last synchronization time to time of this sync ("
<< QDateTime::fromMSecsSinceEpoch(syncBegin).toString("yyyy-MM-dd hh:mm:ss.zzz")
<< ")";
}
qInfo(syncmanager).noquote() << "Update last synchronization time to time of this sync ("
<< QDateTime::fromMSecsSinceEpoch(syncBegin).toString("yyyy-MM-dd hh:mm:ss.zzz")
<< ")";
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "SYNCHRONIZATION FINISHED";
qInfo(syncmanager) << equals;
QCoreApplication::exit();
});
});
processData();
colnod->processData(preparedData->colnodData);
//qCritical(syncmanager) << "Not possible to upload data to colnod";
});
});
return true;
}
std::unique_ptr<PreparedData> SyncManager::prepareDataNew(const std::shared_ptr<Data> &databaseData, const std::shared_ptr<Data> &colnodData)
@ -132,6 +151,13 @@ std::unique_ptr<PreparedData> SyncManager::prepareDataNew(const std::shared_ptr<
QList<QString> conflicts;
QList<QString> alreadySynced;
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "PREPARING DATA -> BEGIN";
qInfo(syncmanager) << dash;
qInfo(syncmanager) << "Resolve conflicts ( changes on same item on both Colnod and DB ):";
qInfo(syncmanager) << dash;
for (const auto &colnodToken : colnodData->updatedTokens) {
// not conflict so add to prepered tokens
@ -309,58 +335,3 @@ std::unique_ptr<PreparedData> SyncManager::prepareDataUsindOld(const std::shared
return preparedData;
}
bool SyncManager::downloadData()
{
bool res = true;
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "DOWNLOADING DATA -> BEGIN";
qInfo(syncmanager) << dash;
qInfo(syncmanager).noquote() << "Modified data since last synchronization (" << mlastSync.toDateString() << "):";
qInfo(syncmanager) << dash;
//connect(colnod.get(), &ColnodAPI::downloadDataFinished, this, &SyncManager::saveColnodData);
//connect(database.get(), &DatabaseAPI::downloadDataFinished, this, &SyncManager::saveDatabaseData);
//database->downloadData();
//colnod->downloadData();
return res;
}
bool SyncManager::processData()
{
qInfo(syncmanager) << equals;
qInfo(syncmanager) << "PROCESSING DATA -> BEGIN";
qInfo(syncmanager) << dash;
qInfo(syncmanager) << "Push changes to both Colnod and Database:";
qInfo(syncmanager) << dash;
database->processData(colnodDataToProcess);
connect(colnod.get(), &ColnodAPI::dataProcessFinished, this, &SyncManager::dataProcessFinished);
colnod->processData(databaseDataToProcess);
return true;
}
void SyncManager::updateLastUpdate(qint64 syncBegin)
{
Timestamp time;
time.set_timestamp(syncBegin);
colnod->getTokensUpdated(time);
database->setLastUpdate(databaseDataToProcess->deletedTokens);
connect(colnod.get(), &ColnodAPI::partOfDataReady, [=](const std::shared_ptr<Data> &data){
database->setLastUpdate(data->updatedTokens);
emit syncFinished();
});
}
void SyncManager::areDataReady()
{
if (++dataAmount == 1) {
qInfo(syncmanager) << "DOWNLOADING DATA -> FINISHED";
qInfo(syncmanager) << equals;
emit dataReady();
}
}

18
src/colnod/syncmanager.h
View File

@ -31,18 +31,9 @@ private:
private:
void initNewToken(const std::shared_ptr<Token>& token);
bool downloadData();
//bool prepareData();
bool processData();
void updateLastUpdate(qint64 syncBegin);
private:
Q_SLOT void areDataReady();
signals:
void dataReady();
void dataProcessFinished();
void syncFinished();
void error();
@ -54,14 +45,5 @@ private:
serversdk::DatabaseManager dbMan;
Timestamp mlastSync;
// std::shared_ptr<Data> colnodData;
// std::shared_ptr<Data> databaseData;
std::shared_ptr<Data> colnodDataToProcess;
std::shared_ptr<Data> databaseDataToProcess;
int dataAmount = 0;
bool mDryMode;
std::function<void()> onError = nullptr;
};

16
test/auto/colnod/tst_colnodapi.cpp
View File

@ -29,9 +29,6 @@ private:
QString getLoginToken(ColnodAPI &colnod);
};
void ColnodTest::login()
{
serversdk::RequestManager reqMan;
@ -54,7 +51,8 @@ void ColnodTest::getTokens()
QSignalSpy spy(login.get(), &serversdk::Request::finished);
spy.wait();
QString token = getLoginToken(colnod);
Timestamp time(QDateTime::currentMSecsSinceEpoch());
Timestamp time;
time.setTimestamp(QDateTime::currentMSecsSinceEpoch());
colnod.getTokensUpdated(time);
}
@ -66,7 +64,7 @@ void ColnodTest::download_data_test()
QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished);
QVERIFY(spy.isValid());
colnod.downloadData(Timestamp(1570190399259));
colnod.downloadData(1570190399259);
QVERIFY(spy.wait());
auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0));
@ -81,8 +79,7 @@ void ColnodTest::download_data_test_now()
serversdk::RequestManager reqMan;
ColnodAPI colnod(&reqMan);
Timestamp time(QDateTime::currentMSecsSinceEpoch());
colnod.downloadData(time);
colnod.downloadData(QDateTime::currentMSecsSinceEpoch());
QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished);
QVERIFY(spy.wait());
auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0));
@ -96,17 +93,14 @@ void ColnodTest::download_data_fail_test()
serversdk::RequestManager reqMan;
ColnodAPI colnod(&reqMan, false, "spaceti", "wrong password");
Timestamp time(0);
QSignalSpy spy(&colnod, &ColnodAPI::downloadDataFinished);
colnod.downloadData(time);
colnod.downloadData(0);
QVERIFY(spy.wait());
auto data = qvariant_cast<std::shared_ptr<Data>>(spy.at(0).at(0));
QVERIFY(!data);
}
QString ColnodTest::getLoginToken(ColnodAPI &colnod)