From c4fd385f828fed587145834b73e6a4d81e6daaea Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Mon, 4 Nov 2024 19:05:44 -0300 Subject: [PATCH 01/17] implementing retry system for pull project --- core/merginapi.cpp | 83 +++++++++++++++++++++++++++++++++++++--------- core/merginapi.h | 15 ++++++++- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index ab5e4a618..376cd5cb0 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -265,6 +265,16 @@ void MerginApi::downloadNextItem( const QString &projectFullName ) DownloadQueueItem item = transaction.downloadQueue.takeFirst(); + QString itemInfo = QStringLiteral( "downloadNextItem : DownloadQueueItem(path: %1, size: %2, version: %3, range: %4-%5, diff: %6, temp: %7)" ) + .arg( item.filePath ) + .arg( item.size ) + .arg( item.version ) + .arg( item.rangeFrom ) + .arg( item.rangeTo ) + .arg( item.downloadDiff ) + .arg( item.tempFileName ); + qDebug() << "Processing item:" << itemInfo; + QUrl url( mApiRoot + QStringLiteral( "/v1/project/raw/" ) + projectFullName ); QUrlQuery query; // Handles special chars in a filePath (e.g prevents to convert "+" sign into a space) @@ -287,7 +297,8 @@ void MerginApi::downloadNextItem( const QString &projectFullName ) } QNetworkReply *reply = mManager.get( request ); - connect( reply, &QNetworkReply::finished, this, &MerginApi::downloadItemReplyFinished ); + connect( reply, &QNetworkReply::finished, this, [this, item]() { downloadItemReplyFinished( item ); } ); + transaction.replyPullItems.insert( reply ); CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Requesting item: " ) + url.toString() + @@ -373,14 +384,12 @@ QString MerginApi::getApiKey( const QString &serverName ) return "not-secret-key"; } -void MerginApi::downloadItemReplyFinished() +void MerginApi::downloadItemReplyFinished( DownloadQueueItem item ) { QNetworkReply *r = qobject_cast( sender() ); Q_ASSERT( r ); - QString projectFullName = r->request().attribute( static_cast( AttrProjectFullName ) ).toString(); QString tempFileName = r->request().attribute( static_cast( AttrTempFileName ) ).toString(); - Q_ASSERT( mTransactionalStatus.contains( projectFullName ) ); TransactionStatus &transaction = mTransactionalStatus[projectFullName]; Q_ASSERT( transaction.replyPullItems.contains( r ) ); @@ -388,13 +397,10 @@ void MerginApi::downloadItemReplyFinished() if ( r->error() == QNetworkReply::NoError ) { QByteArray data = r->readAll(); - CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Downloaded item (%1 bytes)" ).arg( data.size() ) ); - QString tempFolder = getTempProjectDir( projectFullName ); QString tempFilePath = tempFolder + "/" + tempFileName; createPathIfNotExists( tempFilePath ); - // save to a tmp file, assemble at the end QFile file( tempFilePath ); if ( file.open( QIODevice::WriteOnly ) ) @@ -406,13 +412,10 @@ void MerginApi::downloadItemReplyFinished() { CoreUtils::log( "pull " + projectFullName, "Failed to open for writing: " + file.fileName() ); } - transaction.transferedSize += data.size(); emit syncProjectStatusChanged( projectFullName, transaction.transferedSize / transaction.totalSize ); - transaction.replyPullItems.remove( r ); r->deleteLater(); - if ( !transaction.downloadQueue.isEmpty() ) { // one request finished, let's start another one @@ -428,6 +431,42 @@ void MerginApi::downloadItemReplyFinished() // no more requests to start, but there are pending requests - let's do nothing and wait } } + else if ( item.retryCount < transaction.MAX_RETRY_COUNT && isRetryableNetworkError( r ) ) + { + item.retryCount++; + // Put the item back at the front for retry + transaction.downloadQueue.prepend( item ); + + CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Retrying download (attempt %1 of 5)" ).arg( transaction.retryCount ) ); + + QString itemInfo = QStringLiteral( "DownloadQueueItem ( path: %1, size: %2, version: %3, range: %4-%5, diff: %6, temp: %7 )" ) + .arg( item.filePath ) + .arg( item.size ) + .arg( item.version ) + .arg( item.rangeFrom ) + .arg( item.rangeTo ) + .arg( item.downloadDiff ) + .arg( item.tempFileName ); + // qDebug() << "Retrying item:" << itemInfo; + + qDebug() << "Current queue:"; + for ( const DownloadQueueItem &queueItem : transaction.downloadQueue ) + { + QString queueItemInfo = QStringLiteral( "DownloadQueueItem ( path: %1, size: %2, version: %3, range: %4-%5, diff: %6, temp: %7)" ) + .arg( queueItem.filePath ) + .arg( queueItem.size ) + .arg( queueItem.version ) + .arg( queueItem.rangeFrom ) + .arg( queueItem.rangeTo ) + .arg( queueItem.downloadDiff ) + .arg( queueItem.tempFileName ); + qDebug() << queueItemInfo; + } + + downloadNextItem( projectFullName ); + transaction.replyPullItems.remove( r ); + r->deleteLater(); + } else { QString serverMsg = extractServerErrorMsg( r->readAll() ); @@ -439,15 +478,12 @@ void MerginApi::downloadItemReplyFinished() serverMsg = r->errorString(); } CoreUtils::log( "pull " + projectFullName, QStringLiteral( "FAILED - %1. %2" ).arg( r->errorString(), serverMsg ) ); - transaction.replyPullItems.remove( r ); r->deleteLater(); - if ( !transaction.pullItemsAborting ) { // the first failed request will abort all the other pending requests too, and finish pull with error abortPullItems( projectFullName ); - // signal a networking error - we may retry int httpCode = r->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); emit networkErrorOccurred( serverMsg, QStringLiteral( "Mergin API error: downloadFile" ), httpCode, projectFullName ); @@ -3222,8 +3258,8 @@ ProjectDiff MerginApi::compareProjectFiles( /* for ( MerginFile file : oldServerFilesMap ) { - // R-D/L-D - // TODO: need to do anything? + // R-D/L-D + // TODO: need to do anything? } */ @@ -3945,3 +3981,20 @@ DownloadQueueItem::DownloadQueueItem( const QString &fp, qint64 s, int v, qint64 tempFileName = CoreUtils::uuidWithoutBraces( QUuid::createUuid() ); } +bool MerginApi::isRetryableNetworkError( QNetworkReply *reply ) +{ + Q_ASSERT( reply ); + + QNetworkReply::NetworkError err = reply->error(); + int httpCode = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); + + bool isRetryableError = ( err == QNetworkReply::TimeoutError || + err == QNetworkReply::TemporaryNetworkFailureError || + err == QNetworkReply::NetworkSessionFailedError || + err == QNetworkReply::UnknownNetworkError ); + + bool isRetryableHttpCode = ( httpCode == 500 || httpCode == 502 || + httpCode == 503 || httpCode == 504 ); + + return isRetryableError || isRetryableHttpCode; +} diff --git a/core/merginapi.h b/core/merginapi.h index 0e5380bb2..1c1133296 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -109,6 +109,7 @@ struct DownloadQueueItem qint64 rangeTo = -1; //!< what range of bytes to download (-1 if downloading the whole file) bool downloadDiff = false; //!< whether to download just the diff between the previous version and the current one QString tempFileName; //!< relative filename of the temporary file where the downloaded content will be stored + int retryCount = 0; }; @@ -142,6 +143,8 @@ struct TransactionStatus Pull }; + static const int MAX_RETRY_COUNT = 5; //!< maximum number of retry attempts for failed network requests + qreal totalSize = 0; //!< total size (in bytes) of files to be pushed or pulled qint64 transferedSize = 0; //!< size (in bytes) of amount of data transferred so far QString transactionUUID; //!< only for push. Initially dummy non-empty string, after server confirms a valid UUID, on finish/cancel it is empty @@ -166,6 +169,9 @@ struct TransactionStatus QList pushQueue; //!< pending list of files to push (at the end of transaction it is empty) QList pushDiffFiles; //!< these are just diff files for push - we don't remove them when pushing chunks (needed for finalization) + // retry handling + int retryCount = 0; //!< current number of retry attempts for failed network requests + QString projectDir; QByteArray projectMetadata; //!< metadata of the new project (not parsed) bool firstTimeDownload = false; //!< only for update. whether this is first time to download the project (on failure we would also remove the project folder) @@ -657,7 +663,7 @@ class MerginApi: public QObject // Pull slots void pullInfoReplyFinished(); - void downloadItemReplyFinished(); + void downloadItemReplyFinished( DownloadQueueItem item ); void cacheServerConfig(); // Push slots @@ -785,6 +791,13 @@ class MerginApi: public QObject //! Works only when login, password and token is set in UserAuth void refreshAuthToken(); + /** + * Checks if a network error should trigger a retry attempt. + * \param reply Network reply to check for retryable errors + * \returns True if the error should trigger a retry, false otherwise + */ + bool isRetryableNetworkError( QNetworkReply *reply ); + QNetworkRequest getDefaultRequest( bool withAuth = true ); bool projectFileHasBeenUpdated( const ProjectDiff &diff ); From be2075dc800543aaaf2c17dc20659ba7eab288ac Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Tue, 5 Nov 2024 07:39:12 -0300 Subject: [PATCH 02/17] getter and setter methods for QNetworkAccessManager --- app/test/testmerginapi.cpp | 2 +- core/merginapi.cpp | 62 +++++++++++++++++++++++--------------- core/merginapi.h | 14 ++++++++- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 815563bef..58b7bc94e 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2687,7 +2687,7 @@ void TestMerginApi::deleteRemoteProjectNow( MerginApi *api, const QString &proje QUrl url( api->mApiRoot + QStringLiteral( "/v2/projects/%1" ).arg( projectId ) ); request.setUrl( url ); qDebug() << "Trying to delete project " << projectName << ", id: " << projectId << " (" << url << ")"; - QNetworkReply *r = api->mManager.deleteResource( request ); + QNetworkReply *r = api->mManager->deleteResource( request ); QSignalSpy spy( r, &QNetworkReply::finished ); spy.wait( TestUtils::SHORT_REPLY ); diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 376cd5cb0..a25f1eb9f 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -49,6 +49,7 @@ MerginApi::MerginApi( LocalProjectsManager &localProjects, QObject *parent ) , mWorkspaceInfo( new MerginWorkspaceInfo ) , mSubscriptionInfo( new MerginSubscriptionInfo ) , mUserAuth( new MerginUserAuth ) + , mManager( new QNetworkAccessManager( this ) ) { // load cached data if there are any QSettings cache; @@ -199,7 +200,7 @@ QString MerginApi::listProjects( const QString &searchExpression, const QString QString requestId = CoreUtils::uuidWithoutBraces( QUuid::createUuid() ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); CoreUtils::log( "list projects", QStringLiteral( "Requesting: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, [this, requestId]() {this->listProjectsReplyFinished( requestId );} ); @@ -248,7 +249,7 @@ QString MerginApi::listProjectsByName( const QStringList &projectNames ) QString requestId = CoreUtils::uuidWithoutBraces( QUuid::createUuid() ); - QNetworkReply *reply = mManager.post( request, body.toJson() ); + QNetworkReply *reply = mManager->post( request, body.toJson() ); CoreUtils::log( "list projects by name", QStringLiteral( "Requesting: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, [this, requestId]() {this->listProjectsByNameReplyFinished( requestId );} ); @@ -296,7 +297,7 @@ void MerginApi::downloadNextItem( const QString &projectFullName ) request.setRawHeader( "Range", range.toUtf8() ); } - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); connect( reply, &QNetworkReply::finished, this, [this, item]() { downloadItemReplyFinished( item ); } ); transaction.replyPullItems.insert( reply ); @@ -603,7 +604,7 @@ void MerginApi::pushFile( const QString &projectFullName, const QString &transac request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); Q_ASSERT( !transaction.replyPushFile ); - transaction.replyPushFile = mManager.post( request, data ); + transaction.replyPushFile = mManager->post( request, data ); connect( transaction.replyPushFile, &QNetworkReply::finished, this, &MerginApi::pushFileReplyFinished ); CoreUtils::log( "push " + projectFullName, QStringLiteral( "Uploading item: " ) + url.toString() ); @@ -626,7 +627,7 @@ void MerginApi::pushStart( const QString &projectFullName, const QByteArray &jso request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); Q_ASSERT( !transaction.replyPushStart ); - transaction.replyPushStart = mManager.post( request, json ); + transaction.replyPushStart = mManager->post( request, json ); connect( transaction.replyPushStart, &QNetworkReply::finished, this, &MerginApi::pushStartReplyFinished ); CoreUtils::log( "push " + projectFullName, QStringLiteral( "Starting push request: " ) + url.toString() ); @@ -689,7 +690,7 @@ void MerginApi::sendPushCancelRequest( const QString &projectFullName, const QSt request.setRawHeader( "Content-Type", "application/json" ); request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); - QNetworkReply *reply = mManager.post( request, QByteArray() ); + QNetworkReply *reply = mManager->post( request, QByteArray() ); connect( reply, &QNetworkReply::finished, this, &MerginApi::pushCancelReplyFinished ); CoreUtils::log( "push " + projectFullName, QStringLiteral( "Requesting upload transaction cancel: " ) + url.toString() ); } @@ -743,7 +744,7 @@ void MerginApi::pushFinish( const QString &projectFullName, const QString &trans request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); Q_ASSERT( !transaction.replyPushFinish ); - transaction.replyPushFinish = mManager.post( request, QByteArray() ); + transaction.replyPushFinish = mManager->post( request, QByteArray() ); connect( transaction.replyPushFinish, &QNetworkReply::finished, this, &MerginApi::pushFinishReplyFinished ); CoreUtils::log( "push " + projectFullName, QStringLiteral( "Requesting transaction finish: " ) + transactionUUID ); @@ -839,7 +840,7 @@ void MerginApi::authorize( const QString &login, const QString &password ) jsonDoc.setObject( jsonObject ); QByteArray json = jsonDoc.toJson( QJsonDocument::Compact ); - QNetworkReply *reply = mManager.post( request, json ); + QNetworkReply *reply = mManager->post( request, json ); connect( reply, &QNetworkReply::finished, this, &MerginApi::authorizeFinished ); CoreUtils::log( "auth", QStringLiteral( "Requesting authorization: " ) + url.toString() ); } @@ -914,7 +915,7 @@ void MerginApi::registerUser( const QString &username, jsonObject.insert( QStringLiteral( "api_key" ), getApiKey( mApiRoot ) ); jsonDoc.setObject( jsonObject ); QByteArray json = jsonDoc.toJson( QJsonDocument::Compact ); - QNetworkReply *reply = mManager.post( request, json ); + QNetworkReply *reply = mManager->post( request, json ); connect( reply, &QNetworkReply::finished, this, [ = ]() { this->registrationFinished( username, password ); } ); CoreUtils::log( "auth", QStringLiteral( "Requesting registration: " ) + url.toString() ); } @@ -950,7 +951,7 @@ void MerginApi::postRegisterUser( const QString &marketingChannel, const QString jsonObject.insert( QStringLiteral( "subscribe" ), wantsNewsletter ); jsonDoc.setObject( jsonObject ); QByteArray json = jsonDoc.toJson( QJsonDocument::Compact ); - QNetworkReply *reply = mManager.post( request, json ); + QNetworkReply *reply = mManager->post( request, json ); connect( reply, &QNetworkReply::finished, this, [ = ]() { this->postRegistrationFinished(); } ); CoreUtils::log( "auth", QStringLiteral( "Requesting post-registration: " ) + url.toString() ); } @@ -976,7 +977,7 @@ void MerginApi::getUserInfo() QUrl url( urlString ); request.setUrl( url ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); CoreUtils::log( "user info", QStringLiteral( "Requesting user info: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, &MerginApi::getUserInfoFinished ); } @@ -1003,7 +1004,7 @@ void MerginApi::getWorkspaceInfo() QUrl url( urlString ); request.setUrl( url ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); CoreUtils::log( "workspace info", QStringLiteral( "Requesting workspace info: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, &MerginApi::getWorkspaceInfoReplyFinished ); } @@ -1034,7 +1035,7 @@ void MerginApi::getServiceInfo() QUrl url( urlString ); request.setUrl( url ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); connect( reply, &QNetworkReply::finished, this, &MerginApi::getServiceInfoReplyFinished ); @@ -1139,7 +1140,7 @@ bool MerginApi::createProject( const QString &projectNamespace, const QString &p jsonDoc.setObject( jsonObject ); QByteArray json = jsonDoc.toJson( QJsonDocument::Compact ); - QNetworkReply *reply = mManager.post( request, json ); + QNetworkReply *reply = mManager->post( request, json ); connect( reply, &QNetworkReply::finished, this, &MerginApi::createProjectFinished ); CoreUtils::log( "create " + projectFullName, QStringLiteral( "Requesting project creation: " ) + url.toString() ); @@ -1159,7 +1160,7 @@ void MerginApi::deleteProject( const QString &projectNamespace, const QString &p QUrl url( mApiRoot + QStringLiteral( "/v1/project/%1" ).arg( projectFullName ) ); request.setUrl( url ); request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); - QNetworkReply *reply = mManager.deleteResource( request ); + QNetworkReply *reply = mManager->deleteResource( request ); connect( reply, &QNetworkReply::finished, this, [this, informUser]() { this->deleteProjectFinished( informUser );} ); CoreUtils::log( "delete " + projectFullName, QStringLiteral( "Requesting project deletion: " ) + url.toString() ); } @@ -1470,7 +1471,7 @@ QNetworkReply *MerginApi::getProjectInfo( const QString &projectFullName, bool w request.setUrl( url ); request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); - return mManager.get( request ); + return mManager->get( request ); } bool MerginApi::validateAuth() @@ -1689,7 +1690,7 @@ void MerginApi::pingMergin() QUrl url( mApiRoot + QStringLiteral( "/ping" ) ); request.setUrl( url ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); CoreUtils::log( "ping", QStringLiteral( "Requesting: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, &MerginApi::pingMerginReplyFinished ); } @@ -2594,7 +2595,7 @@ void MerginApi::requestServerConfig( const QString &projectFullName ) request.setAttribute( static_cast( AttrProjectFullName ), projectFullName ); Q_ASSERT( !transaction.replyPullServerConfig ); - transaction.replyPullServerConfig = mManager.get( request ); + transaction.replyPullServerConfig = mManager->get( request ); connect( transaction.replyPullServerConfig, &QNetworkReply::finished, this, &MerginApi::cacheServerConfig ); CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Requesting mergin config: " ) + url.toString() ); @@ -3555,7 +3556,7 @@ void MerginApi::deleteAccount() QNetworkRequest request = getDefaultRequest(); QUrl url( mApiRoot + QStringLiteral( "/v1/user" ) ); request.setUrl( url ); - QNetworkReply *reply = mManager.deleteResource( request ); + QNetworkReply *reply = mManager->deleteResource( request ); connect( reply, &QNetworkReply::finished, this, [this]() { this->deleteAccountFinished();} ); CoreUtils::log( "delete account " + mUserAuth->username(), QStringLiteral( "Requesting account deletion: " ) + url.toString() ); } @@ -3607,7 +3608,7 @@ void MerginApi::getServerConfig() QUrl url( urlString ); request.setUrl( url ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); connect( reply, &QNetworkReply::finished, this, &MerginApi::getServerConfigReplyFinished ); CoreUtils::log( "Config", QStringLiteral( "Requesting server configuration: " ) + url.toString() ); @@ -3723,7 +3724,7 @@ void MerginApi::listWorkspaces() QNetworkRequest request = getDefaultRequest( mUserAuth->hasAuthData() ); request.setUrl( url ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); CoreUtils::log( "list workspaces", QStringLiteral( "Requesting: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, &MerginApi::listWorkspacesReplyFinished ); } @@ -3779,7 +3780,7 @@ void MerginApi::listInvitations() QNetworkRequest request = getDefaultRequest( mUserAuth->hasAuthData() ); request.setUrl( url ); - QNetworkReply *reply = mManager.get( request ); + QNetworkReply *reply = mManager->get( request ); CoreUtils::log( "list invitations", QStringLiteral( "Requesting: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, &MerginApi::listInvitationsReplyFinished ); } @@ -3842,7 +3843,7 @@ void MerginApi::processInvitation( const QString &uuid, bool accept ) jsonObject.insert( QStringLiteral( "accept" ), accept ); jsonDoc.setObject( jsonObject ); QByteArray json = jsonDoc.toJson( QJsonDocument::Compact ); - QNetworkReply *reply = mManager.post( request, json ); + QNetworkReply *reply = mManager->post( request, json ); CoreUtils::log( "process invitation", QStringLiteral( "Requesting: " ) + url.toString() ); connect( reply, &QNetworkReply::finished, this, &MerginApi::processInvitationReplyFinished ); } @@ -3904,7 +3905,7 @@ bool MerginApi::createWorkspace( const QString &workspaceName ) jsonDoc.setObject( jsonObject ); QByteArray json = jsonDoc.toJson( QJsonDocument::Compact ); - QNetworkReply *reply = mManager.post( request, json ); + QNetworkReply *reply = mManager->post( request, json ); connect( reply, &QNetworkReply::finished, this, &MerginApi::createWorkspaceReplyFinished ); CoreUtils::log( "create " + workspaceName, QStringLiteral( "Requesting workspace creation: " ) + url.toString() ); @@ -3998,3 +3999,16 @@ bool MerginApi::isRetryableNetworkError( QNetworkReply *reply ) return isRetryableError || isRetryableHttpCode; } + +void MerginApi::setNetworkManager( QNetworkAccessManager *manager ) +{ + if ( mManager == manager ) + return; + + delete mManager; + mManager = manager; + if ( mManager ) + mManager->setParent( this ); + + emit networkManagerChanged(); +} diff --git a/core/merginapi.h b/core/merginapi.h index 1c1133296..e8aa30a1d 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -579,6 +579,17 @@ class MerginApi: public QObject */ bool apiSupportsWorkspaces(); + /** + * Returns the network manager used for Mergin API requests + */ + QNetworkAccessManager *networkManager() const { return mManager; } + + /** + * Sets the network manager to be used for Mergin API requests + * \param QNetworkAccessManager + */ + void setNetworkManager( QNetworkAccessManager *manager ); + signals: void apiSupportsSubscriptionsChanged(); void supportsSelectiveSyncChanged(); @@ -656,6 +667,7 @@ class MerginApi: public QObject void apiSupportsWorkspacesChanged(); void serverWasUpgraded(); + void networkManagerChanged(); private slots: void listProjectsReplyFinished( QString requestId ); @@ -802,7 +814,7 @@ class MerginApi: public QObject bool projectFileHasBeenUpdated( const ProjectDiff &diff ); - QNetworkAccessManager mManager; + QNetworkAccessManager *mManager = nullptr; QString mApiRoot; LocalProjectsManager &mLocalProjects; QString mDataDir; // dir with all projects From 7b10df25ddff4db91a870e619861a471df2bb67b Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Tue, 5 Nov 2024 08:35:04 -0300 Subject: [PATCH 03/17] mock QNetworkAccessManager and QNetworkReply, testDownloadWithNetworkError --- app/test/testmerginapi.cpp | 29 ++++++++++++++++++++++++++++ app/test/testmerginapi.h | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 58b7bc94e..806bda6db 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2942,3 +2942,32 @@ void TestMerginApi::testParseVersion() QCOMPARE( major, 2024 ); QCOMPARE( minor, 4 ); } + +void TestMerginApi::testDownloadWithNetworkError() +{ + MockNetworkManager *mockManager = new MockNetworkManager( mApi ); + mApi->setNetworkManager( mockManager ); + + QString projectName = "testDownloadWithNetworkError"; + QString projectNamespace = mWorkspaceName; + createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); + + mockManager->setShouldFail( true ); + + QSignalSpy spy( mApi, &MerginApi::syncProjectFinished ); + QSignalSpy errorSpy( mApi, &MerginApi::networkErrorOccurred ); + + mApi->pullProject( projectNamespace, projectName ); + + QVERIFY( spy.wait( TestUtils::LONG_REPLY * 5 ) ); + + QCOMPARE( spy.count(), 1 ); + QList arguments = spy.takeFirst(); + QVERIFY( !arguments.at( 1 ).toBool() ); + + QCOMPARE( errorSpy.count(), 1 ); + + mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); + + deleteRemoteProjectNow( mApi, projectNamespace, projectName ); +} diff --git a/app/test/testmerginapi.h b/app/test/testmerginapi.h index 25292fcbd..93f849c9a 100644 --- a/app/test/testmerginapi.h +++ b/app/test/testmerginapi.h @@ -21,6 +21,44 @@ #include +class MockReply : public QNetworkReply +{ + public: + explicit MockReply( QObject *parent = nullptr ) : QNetworkReply( parent ) + { + QNetworkReply::setError( QNetworkReply::ConnectionRefusedError, "Mock network failure" ); + QMetaObject::invokeMethod( this, "finished", Qt::QueuedConnection ); + } + + void abort() override {} + qint64 readData( char *, qint64 ) override { return -1; } + qint64 writeData( const char *, qint64 ) override { return -1; } +}; + +class MockNetworkManager : public QNetworkAccessManager +{ + public: + explicit MockNetworkManager( QObject *parent = nullptr ) + : QNetworkAccessManager( parent ) + , mShouldFail( false ) + {} + + void setShouldFail( bool shouldFail ) { mShouldFail = shouldFail; } + bool shouldFail() const { return mShouldFail; } + + QNetworkReply *post( const QNetworkRequest &request, const QByteArray &data ) + { + if ( mShouldFail ) + { + return new MockReply( this ); + } + return QNetworkAccessManager::post( request, data ); + } + + private: + bool mShouldFail; +}; + class TestMerginApi: public QObject { Q_OBJECT @@ -40,6 +78,7 @@ class TestMerginApi: public QObject void testListProject(); void testListProjectsByName(); void testDownloadProject(); + void testDownloadWithNetworkError(); void testDownloadProjectSpecChars(); void testCancelDownloadProject(); void testCreateProjectTwice(); From 7f198e8b7b45439abd28e4e770117b78354da75e Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Tue, 5 Nov 2024 09:05:35 -0300 Subject: [PATCH 04/17] test --- app/test/testmerginapi.cpp | 32 ++++++++++++++++---------------- app/test/testmerginapi.h | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 806bda6db..9db85300e 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2945,29 +2945,29 @@ void TestMerginApi::testParseVersion() void TestMerginApi::testDownloadWithNetworkError() { - MockNetworkManager *mockManager = new MockNetworkManager( mApi ); - mApi->setNetworkManager( mockManager ); + // MockNetworkManager *mockManager = new MockNetworkManager( mApi ); + // mApi->setNetworkManager( mockManager ); - QString projectName = "testDownloadWithNetworkError"; - QString projectNamespace = mWorkspaceName; - createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); + // QString projectName = "testDownloadWithNetworkError"; + // QString projectNamespace = mWorkspaceName; + // createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); - mockManager->setShouldFail( true ); + // mockManager->setShouldFail( true ); - QSignalSpy spy( mApi, &MerginApi::syncProjectFinished ); - QSignalSpy errorSpy( mApi, &MerginApi::networkErrorOccurred ); + // QSignalSpy spy( mApi, &MerginApi::syncProjectFinished ); + // QSignalSpy errorSpy( mApi, &MerginApi::networkErrorOccurred ); - mApi->pullProject( projectNamespace, projectName ); + // mApi->pullProject( projectNamespace, projectName ); - QVERIFY( spy.wait( TestUtils::LONG_REPLY * 5 ) ); + // QVERIFY( spy.wait( TestUtils::LONG_REPLY * 5 ) ); - QCOMPARE( spy.count(), 1 ); - QList arguments = spy.takeFirst(); - QVERIFY( !arguments.at( 1 ).toBool() ); + // QCOMPARE( spy.count(), 1 ); + // QList arguments = spy.takeFirst(); + // QVERIFY( !arguments.at( 1 ).toBool() ); - QCOMPARE( errorSpy.count(), 1 ); + // QCOMPARE( errorSpy.count(), 1 ); - mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); + // mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); - deleteRemoteProjectNow( mApi, projectNamespace, projectName ); + // deleteRemoteProjectNow( mApi, projectNamespace, projectName ); } diff --git a/app/test/testmerginapi.h b/app/test/testmerginapi.h index 93f849c9a..1f3ea30db 100644 --- a/app/test/testmerginapi.h +++ b/app/test/testmerginapi.h @@ -26,7 +26,7 @@ class MockReply : public QNetworkReply public: explicit MockReply( QObject *parent = nullptr ) : QNetworkReply( parent ) { - QNetworkReply::setError( QNetworkReply::ConnectionRefusedError, "Mock network failure" ); + QNetworkReply::setError( QNetworkReply::UnknownNetworkError, "Mock network failure" ); QMetaObject::invokeMethod( this, "finished", Qt::QueuedConnection ); } From 78a5a00c7b854ab45f8a4cea7b162219978d3ef9 Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Tue, 5 Nov 2024 10:29:08 -0300 Subject: [PATCH 05/17] five attemps per transaction plus debug code removal --- core/merginapi.cpp | 41 +++-------------------------------------- core/merginapi.h | 4 +--- 2 files changed, 4 insertions(+), 41 deletions(-) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index a25f1eb9f..992339903 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -266,16 +266,6 @@ void MerginApi::downloadNextItem( const QString &projectFullName ) DownloadQueueItem item = transaction.downloadQueue.takeFirst(); - QString itemInfo = QStringLiteral( "downloadNextItem : DownloadQueueItem(path: %1, size: %2, version: %3, range: %4-%5, diff: %6, temp: %7)" ) - .arg( item.filePath ) - .arg( item.size ) - .arg( item.version ) - .arg( item.rangeFrom ) - .arg( item.rangeTo ) - .arg( item.downloadDiff ) - .arg( item.tempFileName ); - qDebug() << "Processing item:" << itemInfo; - QUrl url( mApiRoot + QStringLiteral( "/v1/project/raw/" ) + projectFullName ); QUrlQuery query; // Handles special chars in a filePath (e.g prevents to convert "+" sign into a space) @@ -432,38 +422,13 @@ void MerginApi::downloadItemReplyFinished( DownloadQueueItem item ) // no more requests to start, but there are pending requests - let's do nothing and wait } } - else if ( item.retryCount < transaction.MAX_RETRY_COUNT && isRetryableNetworkError( r ) ) + else if ( transaction.retryCount < transaction.MAX_RETRY_COUNT && isRetryableNetworkError( r ) ) { - item.retryCount++; - // Put the item back at the front for retry - transaction.downloadQueue.prepend( item ); + transaction.retryCount++; + transaction.downloadQueue.append( item ); CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Retrying download (attempt %1 of 5)" ).arg( transaction.retryCount ) ); - QString itemInfo = QStringLiteral( "DownloadQueueItem ( path: %1, size: %2, version: %3, range: %4-%5, diff: %6, temp: %7 )" ) - .arg( item.filePath ) - .arg( item.size ) - .arg( item.version ) - .arg( item.rangeFrom ) - .arg( item.rangeTo ) - .arg( item.downloadDiff ) - .arg( item.tempFileName ); - // qDebug() << "Retrying item:" << itemInfo; - - qDebug() << "Current queue:"; - for ( const DownloadQueueItem &queueItem : transaction.downloadQueue ) - { - QString queueItemInfo = QStringLiteral( "DownloadQueueItem ( path: %1, size: %2, version: %3, range: %4-%5, diff: %6, temp: %7)" ) - .arg( queueItem.filePath ) - .arg( queueItem.size ) - .arg( queueItem.version ) - .arg( queueItem.rangeFrom ) - .arg( queueItem.rangeTo ) - .arg( queueItem.downloadDiff ) - .arg( queueItem.tempFileName ); - qDebug() << queueItemInfo; - } - downloadNextItem( projectFullName ); transaction.replyPullItems.remove( r ); r->deleteLater(); diff --git a/core/merginapi.h b/core/merginapi.h index e8aa30a1d..81fcd30e8 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -109,7 +109,6 @@ struct DownloadQueueItem qint64 rangeTo = -1; //!< what range of bytes to download (-1 if downloading the whole file) bool downloadDiff = false; //!< whether to download just the diff between the previous version and the current one QString tempFileName; //!< relative filename of the temporary file where the downloaded content will be stored - int retryCount = 0; }; @@ -143,8 +142,6 @@ struct TransactionStatus Pull }; - static const int MAX_RETRY_COUNT = 5; //!< maximum number of retry attempts for failed network requests - qreal totalSize = 0; //!< total size (in bytes) of files to be pushed or pulled qint64 transferedSize = 0; //!< size (in bytes) of amount of data transferred so far QString transactionUUID; //!< only for push. Initially dummy non-empty string, after server confirms a valid UUID, on finish/cancel it is empty @@ -171,6 +168,7 @@ struct TransactionStatus // retry handling int retryCount = 0; //!< current number of retry attempts for failed network requests + static const int MAX_RETRY_COUNT = 5; //!< maximum number of retry attempts for failed network requests QString projectDir; QByteArray projectMetadata; //!< metadata of the new project (not parsed) From b574a27b09584dbc5d25f8b90f574ee990c2f10d Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Tue, 5 Nov 2024 10:51:16 -0300 Subject: [PATCH 06/17] fixing tests --- app/test/testmerginapi.cpp | 39 +++++++++++++++++++++----------------- core/merginapi.cpp | 2 ++ core/merginapi.h | 2 ++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 9db85300e..4d76c17d4 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2945,29 +2945,34 @@ void TestMerginApi::testParseVersion() void TestMerginApi::testDownloadWithNetworkError() { - // MockNetworkManager *mockManager = new MockNetworkManager( mApi ); - // mApi->setNetworkManager( mockManager ); + QString projectName = "testDownloadWithNetworkError"; + QString projectNamespace = mWorkspaceName; - // QString projectName = "testDownloadWithNetworkError"; - // QString projectNamespace = mWorkspaceName; - // createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); + // Create a test project on server + createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); - // mockManager->setShouldFail( true ); + // Create mock network manager + MockNetworkManager *mockManager = new MockNetworkManager( mApi ); + mApi->setNetworkManager( mockManager ); - // QSignalSpy spy( mApi, &MerginApi::syncProjectFinished ); - // QSignalSpy errorSpy( mApi, &MerginApi::networkErrorOccurred ); + // Track retry signals + QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); - // mApi->pullProject( projectNamespace, projectName ); + // Start download and make network fail after project info + mApi->pullProject( projectNamespace, projectName ); - // QVERIFY( spy.wait( TestUtils::LONG_REPLY * 5 ) ); + QSignalSpy pullStartedSpy( mApi, &MerginApi::pullFilesStarted ); + QVERIFY( pullStartedSpy.wait( TestUtils::SHORT_REPLY ) ); + mockManager->setShouldFail( true ); - // QCOMPARE( spy.count(), 1 ); - // QList arguments = spy.takeFirst(); - // QVERIFY( !arguments.at( 1 ).toBool() ); + // Wait for sync to finish + QSignalSpy syncFinishedSpy( mApi, &MerginApi::syncProjectFinished ); + QVERIFY( syncFinishedSpy.wait( TestUtils::LONG_REPLY ) ); - // QCOMPARE( errorSpy.count(), 1 ); + // Verify we got retry signals + QVERIFY( retrySpy.count() > 0 ); - // mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); - - // deleteRemoteProjectNow( mApi, projectNamespace, projectName ); + // Cleanup + mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); + deleteRemoteProjectNow( mApi, projectNamespace, projectName ); } diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 992339903..4b0822840 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -430,6 +430,8 @@ void MerginApi::downloadItemReplyFinished( DownloadQueueItem item ) CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Retrying download (attempt %1 of 5)" ).arg( transaction.retryCount ) ); downloadNextItem( projectFullName ); + + emit downloadItemRetried( projectFullName, transaction.retryCount ); transaction.replyPullItems.remove( r ); r->deleteLater(); } diff --git a/core/merginapi.h b/core/merginapi.h index 81fcd30e8..b415bebff 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -667,6 +667,8 @@ class MerginApi: public QObject void serverWasUpgraded(); void networkManagerChanged(); + void downloadItemRetried( const QString &projectFullName, int retryCount ); + private slots: void listProjectsReplyFinished( QString requestId ); void listProjectsByNameReplyFinished( QString requestId ); From 9ed8bb474d21af6610b1fbf8dec0bcb02322cf57 Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Tue, 5 Nov 2024 11:04:25 -0300 Subject: [PATCH 07/17] fixing code layout --- app/test/testmerginapi.cpp | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 4d76c17d4..1ba8ff113 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2945,34 +2945,34 @@ void TestMerginApi::testParseVersion() void TestMerginApi::testDownloadWithNetworkError() { - QString projectName = "testDownloadWithNetworkError"; - QString projectNamespace = mWorkspaceName; + QString projectName = "testDownloadWithNetworkError"; + QString projectNamespace = mWorkspaceName; - // Create a test project on server - createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); + // Create a test project on server + createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); - // Create mock network manager - MockNetworkManager *mockManager = new MockNetworkManager( mApi ); - mApi->setNetworkManager( mockManager ); + // Create mock network manager + MockNetworkManager *mockManager = new MockNetworkManager( mApi ); + mApi->setNetworkManager( mockManager ); - // Track retry signals - QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); + // Track retry signals + QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); - // Start download and make network fail after project info - mApi->pullProject( projectNamespace, projectName ); + // Start download and make network fail after project info + mApi->pullProject( projectNamespace, projectName ); - QSignalSpy pullStartedSpy( mApi, &MerginApi::pullFilesStarted ); - QVERIFY( pullStartedSpy.wait( TestUtils::SHORT_REPLY ) ); - mockManager->setShouldFail( true ); + QSignalSpy pullStartedSpy( mApi, &MerginApi::pullFilesStarted ); + QVERIFY( pullStartedSpy.wait( TestUtils::SHORT_REPLY ) ); + mockManager->setShouldFail( true ); - // Wait for sync to finish - QSignalSpy syncFinishedSpy( mApi, &MerginApi::syncProjectFinished ); - QVERIFY( syncFinishedSpy.wait( TestUtils::LONG_REPLY ) ); + // Wait for sync to finish + QSignalSpy syncFinishedSpy( mApi, &MerginApi::syncProjectFinished ); + QVERIFY( syncFinishedSpy.wait( TestUtils::LONG_REPLY ) ); - // Verify we got retry signals - QVERIFY( retrySpy.count() > 0 ); + // Verify we got retry signals + QVERIFY( retrySpy.count() > 0 ); - // Cleanup - mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); - deleteRemoteProjectNow( mApi, projectNamespace, projectName ); + // Cleanup + mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); + deleteRemoteProjectNow( mApi, projectNamespace, projectName ); } From 9e029b18744b35cba51597cd31d47adda182af8f Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Wed, 6 Nov 2024 14:57:18 -0300 Subject: [PATCH 08/17] tests adjustments and fixing --- app/test/testmerginapi.cpp | 59 +++++++++++++++++++++++++------------- app/test/testmerginapi.h | 11 ++++++- core/merginapi.cpp | 2 ++ core/merginapi.h | 1 + 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 1ba8ff113..f083bec50 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2945,34 +2945,53 @@ void TestMerginApi::testParseVersion() void TestMerginApi::testDownloadWithNetworkError() { - QString projectName = "testDownloadWithNetworkError"; - QString projectNamespace = mWorkspaceName; + // Store original network manager to restore it later + QNetworkAccessManager *originalManager = mApi->networkManager(); - // Create a test project on server - createRemoteProject( mApiExtra, projectNamespace, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); + // create a project and do initial setup + QString projectName = "testDownloadWithNetworkError"; + createRemoteProject( mApiExtra, mWorkspaceName, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); - // Create mock network manager - MockNetworkManager *mockManager = new MockNetworkManager( mApi ); + // Set up mock network manager that will fail after first chunk + MockNetworkManager *mockManager = new MockNetworkManager(); mApi->setNetworkManager( mockManager ); - // Track retry signals - QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); + QSignalSpy spyDownloadStarted( mApi, &MerginApi::downloadItemsStarted ); + QSignalSpy spyRetried( mApi, &MerginApi::downloadItemRetried ); + QSignalSpy spyFinished( mApi, &MerginApi::syncProjectFinished ); - // Start download and make network fail after project info - mApi->pullProject( projectNamespace, projectName ); + // Start download + mApi->pullProject( mWorkspaceName, projectName ); - QSignalSpy pullStartedSpy( mApi, &MerginApi::pullFilesStarted ); - QVERIFY( pullStartedSpy.wait( TestUtils::SHORT_REPLY ) ); + // Wait for individual downloads to start + QVERIFY( spyDownloadStarted.wait( TestUtils::SHORT_REPLY ) ); + QCOMPARE( spyDownloadStarted.count(), 1 ); + + // Now simulate network failure mockManager->setShouldFail( true ); - // Wait for sync to finish - QSignalSpy syncFinishedSpy( mApi, &MerginApi::syncProjectFinished ); - QVERIFY( syncFinishedSpy.wait( TestUtils::LONG_REPLY ) ); + // Wait for retries + QVERIFY( spyRetried.wait( TestUtils::LONG_REPLY ) ); + QVERIFY( spyRetried.count() >= 1 ); // Should have at least one retry - // Verify we got retry signals - QVERIFY( retrySpy.count() > 0 ); + QList retryArgs = spyRetried.takeFirst(); + QCOMPARE( retryArgs.at( 0 ).toString(), MerginApi::getFullProjectName( mWorkspaceName, projectName ) ); + QVERIFY( retryArgs.at( 1 ).toInt() >= 1 ); - // Cleanup - mApi->setNetworkManager( new QNetworkAccessManager( mApi ) ); - deleteRemoteProjectNow( mApi, projectNamespace, projectName ); + // Restore normal network behavior to let download finish + mockManager->setShouldFail( false ); + + // Wait for download to finish + QVERIFY( spyFinished.wait( TestUtils::LONG_REPLY ) ); + QCOMPARE( spyFinished.count(), 1 ); + + QList finishArgs = spyFinished.takeFirst(); + QCOMPARE( finishArgs.at( 0 ).toString(), MerginApi::getFullProjectName( mWorkspaceName, projectName ) ); + QVERIFY( finishArgs.at( 1 ).toBool() ); // Should finish successfully + + // Clean up and restore original network manager + deleteLocalProject( mApi, mWorkspaceName, projectName ); + mApi->setNetworkManager( originalManager ); + delete mockManager; } + diff --git a/app/test/testmerginapi.h b/app/test/testmerginapi.h index 1f3ea30db..d7b299906 100644 --- a/app/test/testmerginapi.h +++ b/app/test/testmerginapi.h @@ -26,7 +26,7 @@ class MockReply : public QNetworkReply public: explicit MockReply( QObject *parent = nullptr ) : QNetworkReply( parent ) { - QNetworkReply::setError( QNetworkReply::UnknownNetworkError, "Mock network failure" ); + QNetworkReply::setError( QNetworkReply::TimeoutError, "Mock network failure" ); QMetaObject::invokeMethod( this, "finished", Qt::QueuedConnection ); } @@ -46,6 +46,15 @@ class MockNetworkManager : public QNetworkAccessManager void setShouldFail( bool shouldFail ) { mShouldFail = shouldFail; } bool shouldFail() const { return mShouldFail; } + QNetworkReply *get( const QNetworkRequest &request ) + { + if ( mShouldFail ) + { + return new MockReply( this ); + } + return QNetworkAccessManager::get( request ); + } + QNetworkReply *post( const QNetworkRequest &request, const QByteArray &data ) { if ( mShouldFail ) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 4b0822840..398407d6f 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -2441,6 +2441,8 @@ void MerginApi::startProjectPull( const QString &projectFullName ) } else { + emit downloadItemsStarted(); + while ( transaction.replyPullItems.count() < 5 && !transaction.downloadQueue.isEmpty() ) { downloadNextItem( projectFullName ); diff --git a/core/merginapi.h b/core/merginapi.h index b415bebff..6089a7fd3 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -668,6 +668,7 @@ class MerginApi: public QObject void networkManagerChanged(); void downloadItemRetried( const QString &projectFullName, int retryCount ); + void downloadItemsStarted(); private slots: void listProjectsReplyFinished( QString requestId ); From 1994114975f1bac5959e925ea225c51e0e8d610a Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Fri, 8 Nov 2024 00:13:40 -0300 Subject: [PATCH 09/17] test adjustment --- app/test/testmerginapi.cpp | 70 ++++++++++++++++++++------------------ app/test/testmerginapi.h | 47 ++++++++++++++++--------- 2 files changed, 68 insertions(+), 49 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index f083bec50..bc32dc807 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2945,53 +2945,57 @@ void TestMerginApi::testParseVersion() void TestMerginApi::testDownloadWithNetworkError() { - // Store original network manager to restore it later - QNetworkAccessManager *originalManager = mApi->networkManager(); + QString projectName = "testDownloadRetry"; - // create a project and do initial setup - QString projectName = "testDownloadWithNetworkError"; + // create the project on the server (the content is not important) createRemoteProject( mApiExtra, mWorkspaceName, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); - // Set up mock network manager that will fail after first chunk - MockNetworkManager *mockManager = new MockNetworkManager(); + // Create mock manager - initially not failing + MockNetworkManager *mockManager = new MockNetworkManager( this ); mApi->setNetworkManager( mockManager ); - QSignalSpy spyDownloadStarted( mApi, &MerginApi::downloadItemsStarted ); - QSignalSpy spyRetried( mApi, &MerginApi::downloadItemRetried ); - QSignalSpy spyFinished( mApi, &MerginApi::syncProjectFinished ); + // Create signal spies + QSignalSpy startSpy( mApi, &MerginApi::downloadItemsStarted ); + QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); + QSignalSpy finishSpy( mApi, &MerginApi::syncProjectFinished ); - // Start download - mApi->pullProject( mWorkspaceName, projectName ); + // Connect to downloadItemsStarted to trigger failure when items download start + connect( mApi, &MerginApi::downloadItemsStarted, this, [this]() + { + MockNetworkManager *failingManager = new MockNetworkManager( this ); + failingManager->setShouldFail( true ); + mApi->setNetworkManager( failingManager ); + } ); - // Wait for individual downloads to start - QVERIFY( spyDownloadStarted.wait( TestUtils::SHORT_REPLY ) ); - QCOMPARE( spyDownloadStarted.count(), 1 ); + // Try to download the project + mApi->pullProject( mWorkspaceName, projectName ); - // Now simulate network failure - mockManager->setShouldFail( true ); + // Verify a transaction was created + QCOMPARE( mApi->transactions().count(), 1 ); - // Wait for retries - QVERIFY( spyRetried.wait( TestUtils::LONG_REPLY ) ); - QVERIFY( spyRetried.count() >= 1 ); // Should have at least one retry + // Wait for the download to start and then fail + QVERIFY( startSpy.wait( TestUtils::LONG_REPLY ) ); + QVERIFY( finishSpy.wait( TestUtils::LONG_REPLY ) ); - QList retryArgs = spyRetried.takeFirst(); - QCOMPARE( retryArgs.at( 0 ).toString(), MerginApi::getFullProjectName( mWorkspaceName, projectName ) ); - QVERIFY( retryArgs.at( 1 ).toInt() >= 1 ); + // Verify signals were emitted + QVERIFY( startSpy.count() > 0 ); + QVERIFY( retrySpy.count() > 0 ); + QCOMPARE( finishSpy.count(), 1 ); - // Restore normal network behavior to let download finish - mockManager->setShouldFail( false ); + // Verify that 5 (MAX_RETRY_COUNT) retry attempts were made + int maxRetries = TransactionStatus::MAX_RETRY_COUNT; + QCOMPARE( retrySpy.count(), maxRetries ); - // Wait for download to finish - QVERIFY( spyFinished.wait( TestUtils::LONG_REPLY ) ); - QCOMPARE( spyFinished.count(), 1 ); + // Verify the sync failed + QList arguments = finishSpy.takeFirst(); + QVERIFY( !arguments.at( 1 ).toBool() ); - QList finishArgs = spyFinished.takeFirst(); - QCOMPARE( finishArgs.at( 0 ).toString(), MerginApi::getFullProjectName( mWorkspaceName, projectName ) ); - QVERIFY( finishArgs.at( 1 ).toBool() ); // Should finish successfully + // Verify no local project was created + LocalProject localProject = mApi->localProjectsManager().projectFromMerginName( mWorkspaceName, projectName ); + QVERIFY( !localProject.isValid() ); - // Clean up and restore original network manager - deleteLocalProject( mApi, mWorkspaceName, projectName ); + // Restore the original manager + QNetworkAccessManager *originalManager = new QNetworkAccessManager( this ); mApi->setNetworkManager( originalManager ); - delete mockManager; } diff --git a/app/test/testmerginapi.h b/app/test/testmerginapi.h index d7b299906..929f20ca2 100644 --- a/app/test/testmerginapi.h +++ b/app/test/testmerginapi.h @@ -24,15 +24,32 @@ class MockReply : public QNetworkReply { public: - explicit MockReply( QObject *parent = nullptr ) : QNetworkReply( parent ) + explicit MockReply( const QNetworkRequest &request, QNetworkAccessManager::Operation operation, QObject *parent = nullptr ) + : QNetworkReply( parent ) { - QNetworkReply::setError( QNetworkReply::TimeoutError, "Mock network failure" ); + setRequest( request ); + setOperation( operation ); + setUrl( request.url() ); + + setError( QNetworkReply::TimeoutError, "Mock network failure" ); + + QMetaObject::invokeMethod( this, "errorOccurred", Qt::QueuedConnection, Q_ARG( QNetworkReply::NetworkError, QNetworkReply::TimeoutError ) ); QMetaObject::invokeMethod( this, "finished", Qt::QueuedConnection ); } void abort() override {} - qint64 readData( char *, qint64 ) override { return -1; } - qint64 writeData( const char *, qint64 ) override { return -1; } + + qint64 readData( char *data, qint64 maxlen ) override + { + Q_UNUSED( data ); + Q_UNUSED( maxlen ); + return -1; + } + + qint64 bytesAvailable() const override + { + return 0; + } }; class MockNetworkManager : public QNetworkAccessManager @@ -43,25 +60,23 @@ class MockNetworkManager : public QNetworkAccessManager , mShouldFail( false ) {} - void setShouldFail( bool shouldFail ) { mShouldFail = shouldFail; } - bool shouldFail() const { return mShouldFail; } - - QNetworkReply *get( const QNetworkRequest &request ) + void setShouldFail( bool shouldFail ) { - if ( mShouldFail ) - { - return new MockReply( this ); - } - return QNetworkAccessManager::get( request ); + mShouldFail = shouldFail; } - QNetworkReply *post( const QNetworkRequest &request, const QByteArray &data ) + bool shouldFail() const { return mShouldFail; } + + protected: + QNetworkReply *createRequest( Operation op, const QNetworkRequest &request, QIODevice *outgoingData = nullptr ) override { if ( mShouldFail ) { - return new MockReply( this ); + MockReply *reply = new MockReply( request, op, this ); + return reply; } - return QNetworkAccessManager::post( request, data ); + + return QNetworkAccessManager::createRequest( op, request, outgoingData ); } private: From 216a6b9b848bc5e2b73d95e8830e809b52702129 Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Fri, 8 Nov 2024 16:57:38 -0300 Subject: [PATCH 10/17] post review changes --- app/test/testmerginapi.cpp | 2 +- core/merginapi.cpp | 7 ++++--- core/merginapi.h | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index bc32dc807..bf8a3966a 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2982,7 +2982,7 @@ void TestMerginApi::testDownloadWithNetworkError() QVERIFY( retrySpy.count() > 0 ); QCOMPARE( finishSpy.count(), 1 ); - // Verify that 5 (MAX_RETRY_COUNT) retry attempts were made + // Verify that transaction.MAX_RETRY_COUNT retry attempts were made int maxRetries = TransactionStatus::MAX_RETRY_COUNT; QCOMPARE( retrySpy.count(), maxRetries ); diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 398407d6f..7eaa4bb24 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -427,7 +427,8 @@ void MerginApi::downloadItemReplyFinished( DownloadQueueItem item ) transaction.retryCount++; transaction.downloadQueue.append( item ); - CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Retrying download (attempt %1 of 5)" ).arg( transaction.retryCount ) ); + CoreUtils::log( "pull " + projectFullName, QStringLiteral( "Retrying download (attempt %1 of %2)" ).arg( transaction.retryCount ) + .arg( transaction.MAX_RETRY_COUNT ) ); downloadNextItem( projectFullName ); @@ -3228,8 +3229,8 @@ ProjectDiff MerginApi::compareProjectFiles( /* for ( MerginFile file : oldServerFilesMap ) { - // R-D/L-D - // TODO: need to do anything? + // R-D/L-D + // TODO: need to do anything? } */ diff --git a/core/merginapi.h b/core/merginapi.h index 6089a7fd3..17cbac183 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -584,7 +584,6 @@ class MerginApi: public QObject /** * Sets the network manager to be used for Mergin API requests - * \param QNetworkAccessManager */ void setNetworkManager( QNetworkAccessManager *manager ); From 965c176f84a1feeeef61414bcce0e8e8e9438e9e Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Wed, 13 Nov 2024 19:43:28 -0300 Subject: [PATCH 11/17] asserting manager is not null when setting --- core/merginapi.cpp | 3 +++ core/merginapi.h | 1 + 2 files changed, 4 insertions(+) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 7eaa4bb24..9c9998daa 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -3972,6 +3972,9 @@ bool MerginApi::isRetryableNetworkError( QNetworkReply *reply ) void MerginApi::setNetworkManager( QNetworkAccessManager *manager ) { + if ( !manager ) + return; + if ( mManager == manager ) return; diff --git a/core/merginapi.h b/core/merginapi.h index 17cbac183..ebc4fcb64 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -584,6 +584,7 @@ class MerginApi: public QObject /** * Sets the network manager to be used for Mergin API requests + * Function will return early if manager is null. */ void setNetworkManager( QNetworkAccessManager *manager ); From 0cf8f7628580d352d9b6f09403cfe06bb9c7a681 Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Thu, 14 Nov 2024 08:30:00 -0300 Subject: [PATCH 12/17] fixing tests --- app/test/testmerginapi.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index bf8a3966a..f42417baa 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2994,6 +2994,9 @@ void TestMerginApi::testDownloadWithNetworkError() LocalProject localProject = mApi->localProjectsManager().projectFromMerginName( mWorkspaceName, projectName ); QVERIFY( !localProject.isValid() ); + // Disconnect all signals + disconnect( mApi, &MerginApi::downloadItemsStarted, this, nullptr ); + // Restore the original manager QNetworkAccessManager *originalManager = new QNetworkAccessManager( this ); mApi->setNetworkManager( originalManager ); From 0b0a58088efef7de12e1e960a449e5bff30a7445 Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Wed, 20 Nov 2024 19:26:53 -0300 Subject: [PATCH 13/17] post review additions --- app/test/testmerginapi.cpp | 57 +++++++++++++++----------------------- app/test/testmerginapi.h | 21 ++++++++------ core/merginapi.cpp | 6 ++-- 3 files changed, 39 insertions(+), 45 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index f42417baa..e109ec3fb 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2945,60 +2945,49 @@ void TestMerginApi::testParseVersion() void TestMerginApi::testDownloadWithNetworkError() { - QString projectName = "testDownloadRetry"; + // Store original manager + QNetworkAccessManager *originalManager = mApi->networkManager(); - // create the project on the server (the content is not important) + QString projectName = "testDownloadRetry"; createRemoteProject( mApiExtra, mWorkspaceName, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); // Create mock manager - initially not failing - MockNetworkManager *mockManager = new MockNetworkManager( this ); - mApi->setNetworkManager( mockManager ); + MockNetworkManager *failingManager = new MockNetworkManager( this ); + mApi->setNetworkManager( failingManager ); - // Create signal spies QSignalSpy startSpy( mApi, &MerginApi::downloadItemsStarted ); QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); QSignalSpy finishSpy( mApi, &MerginApi::syncProjectFinished ); - // Connect to downloadItemsStarted to trigger failure when items download start - connect( mApi, &MerginApi::downloadItemsStarted, this, [this]() + // First attempt - TimeoutError + connect( mApi, &MerginApi::downloadItemsStarted, this, [this, failingManager]() { - MockNetworkManager *failingManager = new MockNetworkManager( this ); - failingManager->setShouldFail( true ); - mApi->setNetworkManager( failingManager ); + failingManager->setShouldFail( true, QNetworkReply::TimeoutError ); } ); - // Try to download the project mApi->pullProject( mWorkspaceName, projectName ); - - // Verify a transaction was created - QCOMPARE( mApi->transactions().count(), 1 ); - - // Wait for the download to start and then fail QVERIFY( startSpy.wait( TestUtils::LONG_REPLY ) ); QVERIFY( finishSpy.wait( TestUtils::LONG_REPLY ) ); - // Verify signals were emitted - QVERIFY( startSpy.count() > 0 ); - QVERIFY( retrySpy.count() > 0 ); - QCOMPARE( finishSpy.count(), 1 ); - - // Verify that transaction.MAX_RETRY_COUNT retry attempts were made - int maxRetries = TransactionStatus::MAX_RETRY_COUNT; - QCOMPARE( retrySpy.count(), maxRetries ); + disconnect( mApi, &MerginApi::downloadItemsStarted, this, nullptr ); - // Verify the sync failed - QList arguments = finishSpy.takeFirst(); - QVERIFY( !arguments.at( 1 ).toBool() ); + // Second attempt - TemporaryNetworkFailureError + connect( mApi, &MerginApi::downloadItemsStarted, this, [this, failingManager]() + { + failingManager->setShouldFail( true, QNetworkReply::TemporaryNetworkFailureError ); + } ); - // Verify no local project was created - LocalProject localProject = mApi->localProjectsManager().projectFromMerginName( mWorkspaceName, projectName ); - QVERIFY( !localProject.isValid() ); + mApi->pullProject( mWorkspaceName, projectName ); + QVERIFY( startSpy.wait( TestUtils::LONG_REPLY ) ); + QVERIFY( finishSpy.wait( TestUtils::LONG_REPLY ) ); - // Disconnect all signals disconnect( mApi, &MerginApi::downloadItemsStarted, this, nullptr ); - // Restore the original manager - QNetworkAccessManager *originalManager = new QNetworkAccessManager( this ); + QVERIFY( startSpy.count() > 0 ); + QVERIFY( retrySpy.count() > 0 ); + QCOMPARE( finishSpy.count(), 3 ); + + // Restore original manager and clear old one mApi->setNetworkManager( originalManager ); + delete failingManager; } - diff --git a/app/test/testmerginapi.h b/app/test/testmerginapi.h index 929f20ca2..be7f626ef 100644 --- a/app/test/testmerginapi.h +++ b/app/test/testmerginapi.h @@ -24,17 +24,22 @@ class MockReply : public QNetworkReply { public: - explicit MockReply( const QNetworkRequest &request, QNetworkAccessManager::Operation operation, QObject *parent = nullptr ) + explicit MockReply( const QNetworkRequest &request, QNetworkAccessManager::Operation operation, + QObject *parent = nullptr, QNetworkReply::NetworkError errorCode = QNetworkReply::NoError ) : QNetworkReply( parent ) { setRequest( request ); setOperation( operation ); setUrl( request.url() ); - setError( QNetworkReply::TimeoutError, "Mock network failure" ); + if ( errorCode != QNetworkReply::NoError ) + { + setError( errorCode, "Mock network failure" ); + QMetaObject::invokeMethod( this, "errorOccurred", Qt::QueuedConnection, Q_ARG( QNetworkReply::NetworkError, errorCode ) ); + } - QMetaObject::invokeMethod( this, "errorOccurred", Qt::QueuedConnection, Q_ARG( QNetworkReply::NetworkError, QNetworkReply::TimeoutError ) ); QMetaObject::invokeMethod( this, "finished", Qt::QueuedConnection ); + open( QIODevice::ReadOnly ); } void abort() override {} @@ -58,29 +63,29 @@ class MockNetworkManager : public QNetworkAccessManager explicit MockNetworkManager( QObject *parent = nullptr ) : QNetworkAccessManager( parent ) , mShouldFail( false ) + , mErrorCode( QNetworkReply::NoError ) {} - void setShouldFail( bool shouldFail ) + void setShouldFail( bool shouldFail, QNetworkReply::NetworkError errorCode = QNetworkReply::NoError ) { mShouldFail = shouldFail; + mErrorCode = errorCode; } - bool shouldFail() const { return mShouldFail; } - protected: QNetworkReply *createRequest( Operation op, const QNetworkRequest &request, QIODevice *outgoingData = nullptr ) override { if ( mShouldFail ) { - MockReply *reply = new MockReply( request, op, this ); + auto *reply = new MockReply( request, op, this, mErrorCode ); return reply; } - return QNetworkAccessManager::createRequest( op, request, outgoingData ); } private: bool mShouldFail; + QNetworkReply::NetworkError mErrorCode; }; class TestMerginApi: public QObject diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 9c9998daa..adc32b13a 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -406,12 +406,15 @@ void MerginApi::downloadItemReplyFinished( DownloadQueueItem item ) transaction.transferedSize += data.size(); emit syncProjectStatusChanged( projectFullName, transaction.transferedSize / transaction.totalSize ); transaction.replyPullItems.remove( r ); + r->deleteLater(); + if ( !transaction.downloadQueue.isEmpty() ) { // one request finished, let's start another one downloadNextItem( projectFullName ); } + else if ( transaction.replyPullItems.isEmpty() ) { // nothing else to download and all requests are finished, we're done @@ -3978,10 +3981,7 @@ void MerginApi::setNetworkManager( QNetworkAccessManager *manager ) if ( mManager == manager ) return; - delete mManager; mManager = manager; - if ( mManager ) - mManager->setParent( this ); emit networkManagerChanged(); } From eddef138a8c0a9df904fb8c5ce86ddae413fcbf8 Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Wed, 20 Nov 2024 21:05:08 -0300 Subject: [PATCH 14/17] fixing tests --- app/test/testmerginapi.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index e109ec3fb..bc130331f 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2971,6 +2971,8 @@ void TestMerginApi::testDownloadWithNetworkError() disconnect( mApi, &MerginApi::downloadItemsStarted, this, nullptr ); + failingManager->setShouldFail( false ); + // Second attempt - TemporaryNetworkFailureError connect( mApi, &MerginApi::downloadItemsStarted, this, [this, failingManager]() { From 4d5cd9c05ef193b5df85eb8dd19d27eb7150172f Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Wed, 20 Nov 2024 23:13:50 -0300 Subject: [PATCH 15/17] new test version --- app/test/testmerginapi.cpp | 79 +++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index bc130331f..e75354d88 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -2951,45 +2951,62 @@ void TestMerginApi::testDownloadWithNetworkError() QString projectName = "testDownloadRetry"; createRemoteProject( mApiExtra, mWorkspaceName, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); - // Create mock manager - initially not failing - MockNetworkManager *failingManager = new MockNetworkManager( this ); - mApi->setNetworkManager( failingManager ); - - QSignalSpy startSpy( mApi, &MerginApi::downloadItemsStarted ); - QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); - QSignalSpy finishSpy( mApi, &MerginApi::syncProjectFinished ); + // Errors to test + QList errorsToTest = + { + QNetworkReply::TimeoutError, + QNetworkReply::NetworkSessionFailedError + }; - // First attempt - TimeoutError - connect( mApi, &MerginApi::downloadItemsStarted, this, [this, failingManager]() + foreach ( QNetworkReply::NetworkError networkError, errorsToTest ) { - failingManager->setShouldFail( true, QNetworkReply::TimeoutError ); - } ); + // Create mock manager - initially not failing + MockNetworkManager *failingManager = new MockNetworkManager( this ); + mApi->setNetworkManager( failingManager ); - mApi->pullProject( mWorkspaceName, projectName ); - QVERIFY( startSpy.wait( TestUtils::LONG_REPLY ) ); - QVERIFY( finishSpy.wait( TestUtils::LONG_REPLY ) ); + // Create signal spies + QSignalSpy startSpy( mApi, &MerginApi::downloadItemsStarted ); + QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); + QSignalSpy finishSpy( mApi, &MerginApi::syncProjectFinished ); - disconnect( mApi, &MerginApi::downloadItemsStarted, this, nullptr ); + // Trigger the current network error when download starts + connect( mApi, &MerginApi::downloadItemsStarted, this, [this, failingManager, networkError]() + { + failingManager->setShouldFail( true, networkError ); + } ); - failingManager->setShouldFail( false ); + mApi->pullProject( mWorkspaceName, projectName ); - // Second attempt - TemporaryNetworkFailureError - connect( mApi, &MerginApi::downloadItemsStarted, this, [this, failingManager]() - { - failingManager->setShouldFail( true, QNetworkReply::TemporaryNetworkFailureError ); - } ); + // Verify a transaction was created + QCOMPARE( mApi->transactions().count(), 1 ); - mApi->pullProject( mWorkspaceName, projectName ); - QVERIFY( startSpy.wait( TestUtils::LONG_REPLY ) ); - QVERIFY( finishSpy.wait( TestUtils::LONG_REPLY ) ); + // Wait for download to start and then fail + QVERIFY( startSpy.wait( TestUtils::LONG_REPLY ) ); + QVERIFY( finishSpy.wait( TestUtils::LONG_REPLY ) ); - disconnect( mApi, &MerginApi::downloadItemsStarted, this, nullptr ); + // Verify signals were emitted + QVERIFY( startSpy.count() > 0 ); + QVERIFY( retrySpy.count() > 0 ); + QCOMPARE( finishSpy.count(), 1 ); - QVERIFY( startSpy.count() > 0 ); - QVERIFY( retrySpy.count() > 0 ); - QCOMPARE( finishSpy.count(), 3 ); + // Verify that MAX_RETRY_COUNT retry attempts were made + int maxRetries = TransactionStatus::MAX_RETRY_COUNT; + QCOMPARE( retrySpy.count(), maxRetries ); - // Restore original manager and clear old one - mApi->setNetworkManager( originalManager ); - delete failingManager; + // Verify sync failed + QList arguments = finishSpy.takeFirst(); + QVERIFY( !arguments.at( 1 ).toBool() ); + + // Verify no local project was created + LocalProject localProject = mApi->localProjectsManager().projectFromMerginName( mWorkspaceName, projectName ); + QVERIFY( !localProject.isValid() ); + + // Disconnect all signals + disconnect( mApi, &MerginApi::downloadItemsStarted, this, nullptr ); + + // Clean up + mApi->setNetworkManager( originalManager ); + delete failingManager; + } } + From 273c56965e5b1c1553a986a6e535d9ca00f2847d Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Fri, 22 Nov 2024 11:40:26 -0300 Subject: [PATCH 16/17] adding new errors to isRetryableNetworkError --- core/merginapi.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/merginapi.cpp b/core/merginapi.cpp index adc32b13a..62f2461a2 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -3960,17 +3960,18 @@ bool MerginApi::isRetryableNetworkError( QNetworkReply *reply ) Q_ASSERT( reply ); QNetworkReply::NetworkError err = reply->error(); - int httpCode = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute ).toInt(); bool isRetryableError = ( err == QNetworkReply::TimeoutError || err == QNetworkReply::TemporaryNetworkFailureError || err == QNetworkReply::NetworkSessionFailedError || - err == QNetworkReply::UnknownNetworkError ); - - bool isRetryableHttpCode = ( httpCode == 500 || httpCode == 502 || - httpCode == 503 || httpCode == 504 ); - - return isRetryableError || isRetryableHttpCode; + err == QNetworkReply::UnknownNetworkError || + err == QNetworkReply::RemoteHostClosedError || + err == QNetworkReply::ProxyConnectionClosedError || + err == QNetworkReply::ProxyTimeoutError || + err == QNetworkReply::UnknownProxyError || + err == QNetworkReply::ServiceUnavailableError ); + + return isRetryableError; } void MerginApi::setNetworkManager( QNetworkAccessManager *manager ) From 95afeb6bfa1a95a02f83f150c837f01287bf411f Mon Sep 17 00:00:00 2001 From: VitorVieiraZ Date: Fri, 22 Nov 2024 12:18:57 -0300 Subject: [PATCH 17/17] new test testDownloadWithNetworkErrorRecovery --- app/test/testmerginapi.cpp | 72 ++++++++++++++++++++++++++++++++++++++ app/test/testmerginapi.h | 1 + 2 files changed, 73 insertions(+) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index e75354d88..273cf339b 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -3010,3 +3010,75 @@ void TestMerginApi::testDownloadWithNetworkError() } } +void TestMerginApi::testDownloadWithNetworkErrorRecovery() +{ + // Store original manager + QNetworkAccessManager *originalManager = mApi->networkManager(); + + QString projectName = "testDownloadRetryRecovery"; + createRemoteProject( mApiExtra, mWorkspaceName, projectName, mTestDataPath + "/" + TEST_PROJECT_NAME + "/" ); + + // Create mock manager - initially not failing + MockNetworkManager *failingManager = new MockNetworkManager( this ); + mApi->setNetworkManager( failingManager ); + + // Create signal spies + QSignalSpy startSpy( mApi, &MerginApi::downloadItemsStarted ); + QSignalSpy retrySpy( mApi, &MerginApi::downloadItemRetried ); + QSignalSpy finishSpy( mApi, &MerginApi::syncProjectFinished ); + + // Counter to track retry attempts + int retryCount = 0; + QNetworkReply::NetworkError networkError = QNetworkReply::TimeoutError; + + // Reset network after two retries + connect( mApi, &MerginApi::downloadItemRetried, this, [&retryCount, failingManager, this]() + { + retryCount++; + if ( retryCount == 2 ) + { + failingManager->setShouldFail( false ); + disconnect( mApi, &MerginApi::downloadItemsStarted, nullptr, nullptr ); + disconnect( mApi, &MerginApi::downloadItemRetried, nullptr, nullptr ); + } + } ); + + // Trigger network error when download starts + connect( mApi, &MerginApi::downloadItemsStarted, this, [failingManager, networkError]() + { + failingManager->setShouldFail( true, networkError ); + } ); + + mApi->pullProject( mWorkspaceName, projectName ); + + // Verify a transaction was created + QCOMPARE( mApi->transactions().count(), 1 ); + + // Wait for download to start, retry twice, and then complete successfully + QVERIFY( startSpy.wait( TestUtils::LONG_REPLY ) ); + QVERIFY( finishSpy.wait( TestUtils::LONG_REPLY ) ); + + // Verify signals were emitted + QVERIFY( startSpy.count() > 0 ); + QCOMPARE( retrySpy.count(), 2 ); // Should have exactly 2 retries + QCOMPARE( finishSpy.count(), 1 ); + + // Verify sync succeeded + QList arguments = finishSpy.takeFirst(); + QVERIFY( arguments.at( 1 ).toBool() ); + + // Verify local project was created successfully + LocalProject localProject = mApi->localProjectsManager().projectFromMerginName( mWorkspaceName, projectName ); + QVERIFY( localProject.isValid() ); + + // Verify project files were downloaded correctly + QString projectDir = mApi->projectsPath() + "/" + projectName; + QStringList projectFiles = QDir( projectDir ).entryList( QDir::Files ); + QVERIFY( projectFiles.count() > 0 ); + QVERIFY( projectFiles.contains( "project.qgs" ) ); + + // Clean up + mApi->setNetworkManager( originalManager ); + delete failingManager; +} + diff --git a/app/test/testmerginapi.h b/app/test/testmerginapi.h index be7f626ef..9a9640447 100644 --- a/app/test/testmerginapi.h +++ b/app/test/testmerginapi.h @@ -108,6 +108,7 @@ class TestMerginApi: public QObject void testListProjectsByName(); void testDownloadProject(); void testDownloadWithNetworkError(); + void testDownloadWithNetworkErrorRecovery(); void testDownloadProjectSpecChars(); void testCancelDownloadProject(); void testCreateProjectTwice();