From 4f95e4618fb61710e76ba68c32bff4e9e6bc04ce Mon Sep 17 00:00:00 2001 From: arthomnix Date: Sun, 5 Feb 2023 21:09:16 +0000 Subject: [PATCH] NOISSUE Modrinth exporter: lookup all hashes in one request --- launcher/CMakeLists.txt | 2 + .../modrinth/ModrinthHashLookupRequest.cpp | 124 ++++++++++++++++++ .../modrinth/ModrinthHashLookupRequest.h | 55 ++++++++ .../modrinth/ModrinthInstanceExportTask.cpp | 78 +++++------ .../modrinth/ModrinthInstanceExportTask.h | 10 +- 5 files changed, 219 insertions(+), 50 deletions(-) create mode 100644 launcher/modplatform/modrinth/ModrinthHashLookupRequest.cpp create mode 100644 launcher/modplatform/modrinth/ModrinthHashLookupRequest.h diff --git a/launcher/CMakeLists.txt b/launcher/CMakeLists.txt index e813835d..95db0882 100644 --- a/launcher/CMakeLists.txt +++ b/launcher/CMakeLists.txt @@ -529,6 +529,8 @@ set(MODRINTH_SOURCES modplatform/modrinth/ModrinthPackManifest.h modplatform/modrinth/ModrinthInstanceExportTask.h modplatform/modrinth/ModrinthInstanceExportTask.cpp + modplatform/modrinth/ModrinthHashLookupRequest.h + modplatform/modrinth/ModrinthHashLookupRequest.cpp ) add_unit_test(Index diff --git a/launcher/modplatform/modrinth/ModrinthHashLookupRequest.cpp b/launcher/modplatform/modrinth/ModrinthHashLookupRequest.cpp new file mode 100644 index 00000000..de7e3a79 --- /dev/null +++ b/launcher/modplatform/modrinth/ModrinthHashLookupRequest.cpp @@ -0,0 +1,124 @@ +/* + * Copyright 2023 arthomnix + * + * This source is subject to the Microsoft Public License (MS-PL). + * Please see the COPYING.md file for more information. + */ + +#include +#include +#include "ModrinthHashLookupRequest.h" +#include "BuildConfig.h" +#include "Json.h" + +namespace Modrinth +{ + +HashLookupRequest::HashLookupRequest(QList hashes, QList *output) : NetAction(), m_hashes(hashes), m_output(output) +{ + m_url = "https://api.modrinth.com/v2/version_files"; + m_status = Job_NotStarted; +} + +void HashLookupRequest::startImpl() +{ + finished = false; + m_status = Job_InProgress; + + QNetworkRequest request(m_url); + request.setHeader(QNetworkRequest::UserAgentHeader, BuildConfig.USER_AGENT_UNCACHED); + request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"); + + QJsonObject requestObject; + QJsonArray hashes; + + for (const auto &data : m_hashes) { + hashes.append(data.hash); + } + + requestObject.insert("hashes", hashes); + requestObject.insert("algorithm", QJsonValue("sha512")); + + QNetworkReply *rep = m_network->post(request, QJsonDocument(requestObject).toJson()); + m_reply.reset(rep); + connect(rep, &QNetworkReply::uploadProgress, this, &HashLookupRequest::downloadProgress); + connect(rep, &QNetworkReply::finished, this, &HashLookupRequest::downloadFinished); + connect(rep, &QNetworkReply::errorOccurred, this, &HashLookupRequest::downloadError); +} + +void HashLookupRequest::downloadError(QNetworkReply::NetworkError error) +{ + qCritical() << "Modrinth hash lookup request failed with error" << m_reply->errorString() << "Server reply:\n" << m_reply->readAll(); + if (finished) { + qCritical() << "Double finished ModrinthHashLookupRequest!"; + return; + } + m_status = Job_Failed; + finished = true; + m_reply.reset(); + emit failed(m_index_within_job); +} + +void HashLookupRequest::downloadProgress(qint64 bytesReceived, qint64 bytesTotal) +{ + m_total_progress = bytesTotal; + m_progress = bytesReceived; + emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal); +} + +void HashLookupRequest::downloadFinished() +{ + if (finished) { + qCritical() << "Double finished ModrinthHashLookupRequest!"; + return; + } + + QByteArray data = m_reply->readAll(); + m_reply.reset(); + + try { + auto document = Json::requireDocument(data); + auto rootObject = Json::requireObject(document); + + for (const auto &hashData : m_hashes) { + if (rootObject.contains(hashData.hash)) { + auto versionObject = Json::requireObject(rootObject, hashData.hash); + + auto files = Json::requireIsArrayOf(versionObject, "files"); + + QJsonObject file; + + for (const auto &fileJson : files) { + auto hashes = Json::requireObject(fileJson, "hashes"); + QString sha512 = Json::requireString(hashes, "sha512"); + + if (sha512 == hashData.hash) { + file = fileJson; + } + } + + m_output->append(HashLookupResponseData { + hashData.fileInfo, + true, + file + }); + } else { + m_output->append(HashLookupResponseData { + hashData.fileInfo, + false, + QJsonObject() + }); + } + } + + m_status = Job_Finished; + finished = true; + emit succeeded(m_index_within_job); + } catch (const Json::JsonException &e) { + qCritical() << "Failed to parse Modrinth hash lookup response: " << e.cause(); + m_status = Job_Failed; + finished = true; + emit failed(m_index_within_job); + } +} +} \ No newline at end of file diff --git a/launcher/modplatform/modrinth/ModrinthHashLookupRequest.h b/launcher/modplatform/modrinth/ModrinthHashLookupRequest.h new file mode 100644 index 00000000..a3a803f1 --- /dev/null +++ b/launcher/modplatform/modrinth/ModrinthHashLookupRequest.h @@ -0,0 +1,55 @@ +/* + * Copyright 2023 arthomnix + * + * This source is subject to the Microsoft Public License (MS-PL). + * Please see the COPYING.md file for more information. + */ + +#pragma once + +#include +#include +#include "net/NetAction.h" + +namespace Modrinth +{ + +struct HashLookupData +{ + QFileInfo fileInfo; + QString hash; +}; + +struct HashLookupResponseData +{ + QFileInfo fileInfo; + bool found; + QJsonObject fileJson; +}; + +class HashLookupRequest : public NetAction +{ +public: + using Ptr = shared_qobject_ptr; + + explicit HashLookupRequest(QList hashes, QList *output); + static Ptr make(QList hashes, QList *output) { + return Ptr(new HashLookupRequest(hashes, output)); + } + +protected slots: + void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override; + void downloadError(QNetworkReply::NetworkError error) override; + void downloadFinished() override; + void downloadReadyRead() override {} + +public slots: + void startImpl() override; + +private: + QList m_hashes; + std::shared_ptr> m_output; + bool finished = true; +}; + +} \ No newline at end of file diff --git a/launcher/modplatform/modrinth/ModrinthInstanceExportTask.cpp b/launcher/modplatform/modrinth/ModrinthInstanceExportTask.cpp index 4112c5f7..7f33c4de 100644 --- a/launcher/modplatform/modrinth/ModrinthInstanceExportTask.cpp +++ b/launcher/modplatform/modrinth/ModrinthInstanceExportTask.cpp @@ -16,6 +16,7 @@ #include "ui/dialogs/ModrinthExportDialog.h" #include "JlCompress.h" #include "FileSystem.h" +#include "ModrinthHashLookupRequest.h" namespace Modrinth { @@ -76,6 +77,10 @@ void InstanceExportTask::executeTask() m_netJob = new NetJob(tr("Modrinth pack export"), APPLICATION->network()); + QList hashes; + + qint64 progress = 0; + setProgress(progress, filesToResolve.length()); for (const QString &filePath: filesToResolve) { qDebug() << "Attempting to resolve file hash from Modrinth API: " << filePath; QFile file(filePath); @@ -86,20 +91,20 @@ void InstanceExportTask::executeTask() hasher.addData(contents); QString hash = hasher.result().toHex(); - m_responses.append(HashLookupData{ - QFileInfo(file), - hash, - QByteArray() + hashes.append(HashLookupData { + QFileInfo(file), + hash }); - m_netJob->addNetAction(Net::Download::makeByteArray( - QString("https://api.modrinth.com/v2/version_file/%1?algorithm=sha512").arg(hash), - &m_responses.last().response, - Net::Download::Options(Net::Download::Option::AllowNotFound) - )); + progress++; + setProgress(progress, filesToResolve.length()); } } + m_response.reset(new QList); + + m_netJob->addNetAction(HashLookupRequest::make(hashes, m_response.get())); + connect(m_netJob.get(), &NetJob::succeeded, this, &InstanceExportTask::lookupSucceeded); connect(m_netJob.get(), &NetJob::failed, this, &InstanceExportTask::lookupFailed); connect(m_netJob.get(), &NetJob::progress, this, &InstanceExportTask::lookupProgress); @@ -114,43 +119,32 @@ void InstanceExportTask::lookupSucceeded() QList resolvedFiles; QFileInfoList failedFiles; - for (const auto &data : m_responses) { - try { - auto document = Json::requireDocument(data.response); - auto object = Json::requireObject(document); - auto files = Json::requireIsArrayOf(object, "files"); + for (const auto &file : *m_response) { + if (file.found) { + try { + auto url = Json::requireString(file.fileJson, "url"); + auto hashes = Json::requireObject(file.fileJson, "hashes"); - QJsonObject file; + QString sha512Hash = Json::requireString(hashes, "sha512"); + QString sha1Hash = Json::requireString(hashes, "sha1"); - for (const auto &fileJson : files) { - auto hashes = Json::requireObject(fileJson, "hashes"); - QString sha512 = Json::requireString(hashes, "sha512"); + ExportFile fileData; - if (sha512 == data.sha512) { - file = fileJson; - } + QDir gameDir(m_instance->gameRoot()); + + fileData.path = gameDir.relativeFilePath(file.fileInfo.absoluteFilePath()); + fileData.download = url; + fileData.sha512 = sha512Hash; + fileData.sha1 = sha1Hash; + fileData.fileSize = file.fileInfo.size(); + + resolvedFiles << fileData; + } catch (const Json::JsonException &e) { + qDebug() << "File " << file.fileInfo.absoluteFilePath() << " failed to process for reason " << e.cause() << ", adding to overrides"; + failedFiles << file.fileInfo; } - - auto url = Json::requireString(file, "url"); - auto hashes = Json::requireObject(file, "hashes"); - - QString sha512Hash = Json::requireString(hashes, "sha512"); - QString sha1Hash = Json::requireString(hashes, "sha1"); - - ExportFile fileData; - - QDir gameDir(m_instance->gameRoot()); - - fileData.path = gameDir.relativeFilePath(data.fileInfo.absoluteFilePath()); - fileData.download = url; - fileData.sha512 = sha512Hash; - fileData.sha1 = sha1Hash; - fileData.fileSize = data.fileInfo.size(); - - resolvedFiles << fileData; - } catch (const Json::JsonException &e) { - qDebug() << "File " << data.fileInfo.absoluteFilePath() << " failed to process for reason " << e.cause() << ", adding to overrides"; - failedFiles << data.fileInfo; + } else { + failedFiles << file.fileInfo; } } diff --git a/launcher/modplatform/modrinth/ModrinthInstanceExportTask.h b/launcher/modplatform/modrinth/ModrinthInstanceExportTask.h index c1592bc2..4a6010c8 100644 --- a/launcher/modplatform/modrinth/ModrinthInstanceExportTask.h +++ b/launcher/modplatform/modrinth/ModrinthInstanceExportTask.h @@ -11,6 +11,7 @@ #include "BaseInstance.h" #include "net/NetJob.h" #include "ui/dialogs/ModrinthExportDialog.h" +#include "ModrinthHashLookupRequest.h" namespace Modrinth { @@ -35,13 +36,6 @@ struct ExportSettings QString exportPath; }; -struct HashLookupData -{ - QFileInfo fileInfo; - QString sha512; - QByteArray response; -}; - // Using the existing Modrinth::File struct from the importer doesn't actually make much sense here (doesn't support multiple hashes, hash is a byte array rather than a string, no file size, etc) struct ExportFile { @@ -71,7 +65,7 @@ private slots: private: InstancePtr m_instance; ExportSettings m_settings; - QList m_responses; + std::shared_ptr> m_response; NetJob::Ptr m_netJob; };