From c3493d695dfd00d09bf1d5f99e05d1c94d09acac Mon Sep 17 00:00:00 2001 From: JC <8765278+Crzyrndm@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:51:50 +1200 Subject: [PATCH 1/4] parallel operations for some binary caching providers #38404 --- src/vcpkg/binarycaching.cpp | 50 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 3559fd4275..677057f248 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -988,38 +989,38 @@ namespace void acquire_zips(View actions, Span> out_zip_paths) const override { - for (size_t idx = 0; idx < actions.size(); ++idx) - { - auto&& action = *actions[idx]; - const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); - auto tmp = make_temp_archive_path(m_buildtrees, action.spec); - auto res = m_tool->download_file(make_object_path(m_prefix, abi), tmp); - if (res) - { - out_zip_paths[idx].emplace(std::move(tmp), RemoveWhen::always); - } - else - { - out_sink.println_warning(res.error()); - } - } + parallel_transform( + actions, out_zip_paths.begin(), [&](const InstallPlanAction* plan) -> Optional { + auto&& action = *plan; + const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); + auto tmp = make_temp_archive_path(m_buildtrees, action.spec); + auto res = m_tool->download_file(make_object_path(m_prefix, abi), tmp); + if (res) + { + return ZipResource{ZipResource(std::move(tmp), RemoveWhen::always)}; + } + else + { + out_sink.println_warning(res.error()); + return nullopt; + } + }); } void precheck(View actions, Span cache_status) const override { - for (size_t idx = 0; idx < actions.size(); ++idx) - { - auto&& action = *actions[idx]; + parallel_transform(actions, cache_status.begin(), [&](const InstallPlanAction* plan) { + auto&& action = *plan; const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); if (m_tool->stat(make_object_path(m_prefix, abi))) { - cache_status[idx] = CacheAvailability::available; + return CacheAvailability::available; } else { - cache_status[idx] = CacheAvailability::unavailable; + return CacheAvailability::unavailable; } - } + }); } LocalizedString restored_message(size_t count, @@ -1048,9 +1049,8 @@ namespace { if (!request.zip_path) return 0; const auto& zip_path = *request.zip_path.get(); - size_t upload_count = 0; - for (const auto& prefix : m_prefixes) - { + std::atomic_size_t upload_count = 0; + parallel_for_each(m_prefixes, [&](std::string const& prefix) { auto res = m_tool->upload_file(make_object_path(prefix, request.package_abi), zip_path); if (res) { @@ -1060,7 +1060,7 @@ namespace { msg_sink.println_warning(res.error()); } - } + }); return upload_count; } From 6e386ed4bf5bf061fbc5c0f1d017066c44442be7 Mon Sep 17 00:00:00 2001 From: JC <8765278+Crzyrndm@users.noreply.github.com> Date: Thu, 16 May 2024 08:57:01 +1200 Subject: [PATCH 2/4] revert changes to upload path this is pushing to the different caches(?) in parallel, not by package like the others --- src/vcpkg/binarycaching.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 677057f248..76661b4e76 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -1049,8 +1049,9 @@ namespace { if (!request.zip_path) return 0; const auto& zip_path = *request.zip_path.get(); - std::atomic_size_t upload_count = 0; - parallel_for_each(m_prefixes, [&](std::string const& prefix) { + size_t upload_count = 0; + for (const auto& prefix : m_prefixes) + { auto res = m_tool->upload_file(make_object_path(prefix, request.package_abi), zip_path); if (res) { @@ -1060,7 +1061,7 @@ namespace { msg_sink.println_warning(res.error()); } - }); + } return upload_count; } From e4549273fd00a7161e15eb261872077efcd6ec58 Mon Sep 17 00:00:00 2001 From: JC <8765278+Crzyrndm@users.noreply.github.com> Date: Tue, 28 May 2024 10:51:04 +1200 Subject: [PATCH 3/4] remove the variable rename --- src/vcpkg/binarycaching.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 76661b4e76..20643562ff 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -991,9 +991,8 @@ namespace { parallel_transform( actions, out_zip_paths.begin(), [&](const InstallPlanAction* plan) -> Optional { - auto&& action = *plan; - const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); - auto tmp = make_temp_archive_path(m_buildtrees, action.spec); + const auto& abi = plan->package_abi().value_or_exit(VCPKG_LINE_INFO); + auto tmp = make_temp_archive_path(m_buildtrees, plan->spec); auto res = m_tool->download_file(make_object_path(m_prefix, abi), tmp); if (res) { @@ -1010,8 +1009,7 @@ namespace void precheck(View actions, Span cache_status) const override { parallel_transform(actions, cache_status.begin(), [&](const InstallPlanAction* plan) { - auto&& action = *plan; - const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); + const auto& abi = plan->package_abi().value_or_exit(VCPKG_LINE_INFO); if (m_tool->stat(make_object_path(m_prefix, abi))) { return CacheAvailability::available; From d94ca2c852d355c7ab3220443911f1a5de305617 Mon Sep 17 00:00:00 2001 From: JC <8765278+Crzyrndm@users.noreply.github.com> Date: Tue, 28 May 2024 10:51:22 +1200 Subject: [PATCH 4/4] add comment noting calls will occur in parallel --- src/vcpkg/binarycaching.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 20643562ff..4385266eac 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -962,7 +962,11 @@ namespace virtual LocalizedString restored_message(size_t count, std::chrono::high_resolution_clock::duration elapsed) const = 0; + /// checks if package abi is present in this storage + /// NOTE: Operation must be thread safe. Multiple calls will occur in parallel virtual ExpectedL stat(StringView url) const = 0; + /// download a package zip from this storage + /// NOTE: Operation must be thread safe. Multiple calls will occur in parallel virtual ExpectedL download_file(StringView object, const Path& archive) const = 0; virtual ExpectedL upload_file(StringView object, const Path& archive) const = 0; };