From f4a02e3c47c4281fb7f5ef1252fa17113d4482e0 Mon Sep 17 00:00:00 2001 From: Vincent Le Garrec Date: Fri, 22 Nov 2024 16:12:05 +0100 Subject: [PATCH 1/7] Add filesystem-from-git in configuration --- include/vcpkg/base/contractual-constants.h | 1 + include/vcpkg/base/message-data.inc.h | 1 + src/vcpkg/configuration.cpp | 36 +++++++++++++++++++++- src/vcpkg/registries.cpp | 8 +++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h index 1dd7d6fbc9..2dd7a6867d 100644 --- a/include/vcpkg/base/contractual-constants.h +++ b/include/vcpkg/base/contractual-constants.h @@ -49,6 +49,7 @@ namespace vcpkg inline constexpr StringLiteral JsonIdFeatures = "features"; inline constexpr StringLiteral JsonIdFiles = "files"; inline constexpr StringLiteral JsonIdFilesystem = "filesystem"; + inline constexpr StringLiteral JsonIdFilesystemFromGit = "filesystem-from-git"; inline constexpr StringLiteral JsonIdGit = "git"; inline constexpr StringLiteral JsonIdGitTree = "git-tree"; inline constexpr StringLiteral JsonIdHomepage = "homepage"; diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 3131761a91..f71da81514 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -125,6 +125,7 @@ DECLARE_MESSAGE(ADictionaryOfContacts, (), "", "a dictionary of contacts") DECLARE_MESSAGE(AFeature, (), "", "a feature") DECLARE_MESSAGE(AFeatureName, (), "", "a feature name") DECLARE_MESSAGE(AFilesystemRegistry, (), "", "a filesystem registry") +DECLARE_MESSAGE(AFilesystemRegistryFromGit, (), "", "a filesystem registry from git") DECLARE_MESSAGE(AGitObjectSha, (), "", "a git object SHA") DECLARE_MESSAGE(AGitReference, (), "", "a git reference (for example, a branch)") DECLARE_MESSAGE(AGitRegistry, (), "", "a git registry") diff --git a/src/vcpkg/configuration.cpp b/src/vcpkg/configuration.cpp index 9aa32cd370..70b8e4ea80 100644 --- a/src/vcpkg/configuration.cpp +++ b/src/vcpkg/configuration.cpp @@ -235,6 +235,28 @@ namespace r.check_for_unexpected_fields(obj, valid_filesystem_fields, msg::format(msgAFilesystemRegistry)); } + else if (kind == JsonIdFilesystemFromGit) + { + r.required_object_field(msg::format(msgAFilesystemRegistryFromGit), + obj, + JsonIdRepository, + res.repo.emplace(), + GitUrlDeserializer::instance); + + if (!r.optional_object_field( + obj, JsonIdReference, res.reference.emplace(), GitReferenceDeserializer::instance)) + { + res.reference = nullopt; + } + + r.required_object_field(msg::format(msgAFilesystemRegistryFromGit), + obj, + JsonIdBaseline, + res.baseline.emplace(), + BaselineShaDeserializer::instance); + + r.check_for_unexpected_fields(obj, valid_git_fields, msg::format(msgAFilesystemRegistryFromGit)); + } else if (kind == JsonIdGit) { r.required_object_field( @@ -272,7 +294,8 @@ namespace } else { - StringLiteral valid_kinds[] = {JsonIdBuiltin, JsonIdFilesystem, JsonIdGit, JsonIdArtifact}; + StringLiteral valid_kinds[] = { + JsonIdBuiltin, JsonIdFilesystem, JsonIdFilesystemFromGit, JsonIdGit, JsonIdArtifact}; r.add_generic_error(type_name(), msg::format(msgFieldKindDidNotHaveExpectedValue, msg::expected = Strings::join(", ", valid_kinds), @@ -753,6 +776,10 @@ namespace vcpkg { return path.value_or_exit(VCPKG_LINE_INFO); } + if (kind == JsonIdFilesystemFromGit) + { + return repo.value_or_exit(VCPKG_LINE_INFO); + } if (kind == JsonIdGit) { return repo.value_or_exit(VCPKG_LINE_INFO); @@ -902,6 +929,13 @@ namespace vcpkg config.reference.value_or("HEAD"), config.baseline.value_or_exit(VCPKG_LINE_INFO)); } + else if (*k == JsonIdFilesystemFromGit) + { + return make_filesystem_from_git_registry(paths, + config.repo.value_or_exit(VCPKG_LINE_INFO), + config.reference.value_or("HEAD"), + config.baseline.value_or_exit(VCPKG_LINE_INFO)); + } else if (*k == JsonIdFilesystem) { return make_filesystem_registry(paths.get_filesystem(), diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index c77c0f75dc..f7e1cbb60f 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -1744,6 +1744,14 @@ namespace vcpkg { return std::make_unique(paths, std::move(repo), std::move(reference), std::move(baseline)); } + std::unique_ptr make_filesystem_from_git_registry(const VcpkgPaths& paths, + std::string repo, + std::string reference, + std::string baseline) + { + return std::make_unique( + paths, std::move(repo), std::move(reference), std::move(baseline)); + } std::unique_ptr make_filesystem_registry(const ReadOnlyFilesystem& fs, Path path, std::string baseline) From 7c4a7bb97bd6a32650c379d34f9de8963aaf738c Mon Sep 17 00:00:00 2001 From: Vincent Le Garrec Date: Fri, 22 Nov 2024 16:12:14 +0100 Subject: [PATCH 2/7] Add filesystem-from-git in schema --- docs/vcpkg-schema-definitions.schema.json | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/vcpkg-schema-definitions.schema.json b/docs/vcpkg-schema-definitions.schema.json index e18d70d631..8764ee7250 100644 --- a/docs/vcpkg-schema-definitions.schema.json +++ b/docs/vcpkg-schema-definitions.schema.json @@ -132,6 +132,7 @@ "enum": [ "builtin", "git", + "filesystem-from-git", "filesystem" ] }, @@ -146,6 +147,9 @@ { "$ref": "#/definitions/git-registry" }, + { + "$ref": "#/definitions/filesystem-from-git-registry" + }, { "$ref": "#/definitions/filesystem-registry" } @@ -650,6 +654,56 @@ }, "else": false }, + "filesystem-from-git-registry": { + "type": "object", + "description": "A port registry backed by a filesystem layout and a git repo.", + "if": { + "properties": { + "kind": { + "type": "string", + "enum": [ + "filesystem-from-git" + ] + } + }, + "required": [ + "kind" + ] + }, + "then": { + "properties": { + "kind": { + "type": "string", + "enum": [ + "filesystem-from-git" + ] + }, + "repository": { + "type": "string", + "description": "The URI or SSH entry to clone from. (as if by `git clone`)" + }, + "baseline": { + "description": "The SHA that will be checked out to find default versions for ports from this registry.", + "$ref": "#/definitions/sha1" + }, + "reference": { + "type": "string", + "description": "The ref to fetch when cloning this git registry." + }, + "packages": true + }, + "required": [ + "kind", + "repository", + "baseline" + ], + "patternProperties": { + "^\\$": {} + }, + "additionalProperties": false + }, + "else": false + }, "has-one-version": { "type": "object", "oneOf": [ @@ -846,6 +900,7 @@ "enum": [ "builtin", "git", + "filesystem-from-git", "filesystem", "artifact" ] @@ -880,6 +935,9 @@ { "$ref": "#/definitions/git-registry" }, + { + "$ref": "#/definitions/filesystem-from-git-registry" + }, { "$ref": "#/definitions/filesystem-registry" } From 16f22747076bfd77c46f04fa8b10f9f6b723de50 Mon Sep 17 00:00:00 2001 From: Vincent Le Garrec Date: Fri, 22 Nov 2024 16:12:22 +0100 Subject: [PATCH 3/7] Duplicate GitRegistry to FilesystemFromGitRegistry --- include/vcpkg/registries.h | 4 + src/vcpkg/registries.cpp | 415 +++++++++++++++++++++++++++++++++++++ 2 files changed, 419 insertions(+) diff --git a/include/vcpkg/registries.h b/include/vcpkg/registries.h index 66d8886623..315925e0eb 100644 --- a/include/vcpkg/registries.h +++ b/include/vcpkg/registries.h @@ -156,6 +156,10 @@ namespace vcpkg std::string repo, std::string reference, std::string baseline); + std::unique_ptr make_filesystem_from_git_registry(const VcpkgPaths& paths, + std::string repo, + std::string reference, + std::string baseline); std::unique_ptr make_filesystem_registry(const ReadOnlyFilesystem& fs, Path path, std::string baseline); diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index f7e1cbb60f..b966b3d8c7 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -399,6 +399,172 @@ namespace DelayedInit> m_baseline; }; + struct FilesystemFromGitRegistry; + + struct FilesystemFromGitRegistryEntry final : RegistryEntry + { + FilesystemFromGitRegistryEntry(StringView port_name, + const FilesystemFromGitRegistry& parent, + bool stale, + std::vector&& version_entries); + + ExpectedL> get_port_versions() const override; + ExpectedL try_load_port(const Version& version) const override; + + private: + ExpectedL ensure_not_stale() const; + + std::string port_name; + + const FilesystemFromGitRegistry& parent; + + // Indicates whether port_versions and git_trees were filled in with stale (i.e. lock) data. + mutable bool stale; + + mutable PortVersionsGitTreesStructOfArrays last_loaded; + }; + + struct FilesystemFromGitRegistry final : RegistryImplementation + { + FilesystemFromGitRegistry(const VcpkgPaths& paths, + std::string&& repo, + std::string&& reference, + std::string&& baseline) + : m_paths(paths) + , m_repo(std::move(repo)) + , m_reference(std::move(reference)) + , m_baseline_identifier(std::move(baseline)) + { + } + + StringLiteral kind() const override { return "filesystem-from-git"; } + + ExpectedL> get_port_entry(StringView) const override; + + ExpectedL append_all_port_names(std::vector&) const override; + + ExpectedL try_append_all_port_names_no_network(std::vector& port_names) const override; + + ExpectedL> get_baseline_version(StringView) const override; + + private: + friend FilesystemFromGitRegistryEntry; + + const ExpectedL& get_lock_entry() const + { + return m_lock_entry.get( + [this]() { return m_paths.get_installed_lockfile().get_or_fetch(m_paths, m_repo, m_reference); }); + } + + const ExpectedL& get_versions_tree_path() const + { + return m_versions_tree.get([this]() -> ExpectedL { + auto& maybe_lock_entry = get_lock_entry(); + auto lock_entry = maybe_lock_entry.get(); + if (!lock_entry) + { + return maybe_lock_entry.error(); + } + + auto maybe_up_to_date = lock_entry->ensure_up_to_date(m_paths); + if (!maybe_up_to_date) + { + return std::move(maybe_up_to_date).error(); + } + + auto maybe_tree = m_paths.git_find_object_id_for_remote_registry_path(lock_entry->commit_id(), + FileVersions.to_string()); + auto tree = maybe_tree.get(); + if (!tree) + { + get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorNoVersionsAtCommit); + return msg::format_error(msgCouldNotFindGitTreeAtCommit, + msg::package_name = m_repo, + msg::commit_sha = lock_entry->commit_id()) + .append_raw('\n') + .append_raw(maybe_tree.error()); + } + + auto maybe_path = m_paths.git_extract_tree_from_remote_registry(*tree); + auto path = maybe_path.get(); + if (!path) + { + return msg::format_error(msgFailedToCheckoutRepo, msg::package_name = m_repo) + .append_raw('\n') + .append(maybe_path.error()); + } + + return std::move(*path); + }); + } + + struct VersionsTreePathResult + { + Path p; + bool stale; + }; + + ExpectedL get_unstale_stale_versions_tree_path() const + { + auto& maybe_versions_tree = get_versions_tree_path(); + if (auto versions_tree = maybe_versions_tree.get()) + { + return VersionsTreePathResult{*versions_tree, false}; + } + + return maybe_versions_tree.error(); + } + + ExpectedL get_stale_versions_tree_path() const + { + const auto& maybe_entry = get_lock_entry(); + auto entry = maybe_entry.get(); + if (!entry) + { + return maybe_entry.error(); + } + + if (!entry->stale()) + { + return get_unstale_stale_versions_tree_path(); + } + + if (!m_stale_versions_tree.has_value()) + { + auto maybe_tree = + m_paths.git_find_object_id_for_remote_registry_path(entry->commit_id(), FileVersions.to_string()); + auto tree = maybe_tree.get(); + if (!tree) + { + // This could be caused by git gc or otherwise -- fall back to full fetch + return get_unstale_stale_versions_tree_path(); + } + + auto maybe_path = m_paths.git_extract_tree_from_remote_registry(*tree); + auto path = maybe_path.get(); + if (!path) + { + // This could be caused by git gc or otherwise -- fall back to full fetch + return get_unstale_stale_versions_tree_path(); + } + + m_stale_versions_tree = std::move(*path); + } + + return VersionsTreePathResult{m_stale_versions_tree.value_or_exit(VCPKG_LINE_INFO), true}; + } + + const VcpkgPaths& m_paths; + + std::string m_repo; + std::string m_reference; + std::string m_baseline_identifier; + DelayedInit> m_lock_entry; + mutable Optional m_stale_versions_tree; + DelayedInit> m_versions_tree; + DelayedInit> m_baseline; + }; + struct BuiltinPortTreeRegistryEntry final : RegistryEntry { BuiltinPortTreeRegistryEntry(const SourceControlFileAndLocation& load_result_) : load_result(load_result_) { } @@ -877,6 +1043,172 @@ namespace } // } FilesystemRegistry::RegistryImplementation + // { FilesystemFromGitRegistry::RegistryImplementation + ExpectedL> FilesystemFromGitRegistry::get_port_entry(StringView port_name) const + { + auto maybe_stale_vtp = get_stale_versions_tree_path(); + auto stale_vtp = maybe_stale_vtp.get(); + if (!stale_vtp) + { + return std::move(maybe_stale_vtp).error(); + } + + { + // try to load using "stale" version database + auto maybe_maybe_version_entries = + load_git_versions_file(m_paths.get_filesystem(), stale_vtp->p, port_name).entries; + auto maybe_version_entries = maybe_maybe_version_entries.get(); + if (!maybe_version_entries) + { + return std::move(maybe_maybe_version_entries).error(); + } + + auto version_entries = maybe_version_entries->get(); + if (version_entries) + { + return std::make_unique( + port_name, *this, stale_vtp->stale, std::move(*version_entries)); + } + } + + if (!stale_vtp->stale) + { + // data is already live but we don't know of this port + return std::unique_ptr(); + } + + return get_versions_tree_path().then([this, &port_name](const Path& live_vcb) { + return load_git_versions_file(m_paths.get_filesystem(), live_vcb, port_name) + .entries.then([this, &port_name](Optional>&& maybe_version_entries) + -> ExpectedL> { + auto version_entries = maybe_version_entries.get(); + if (!version_entries) + { + // data is already live but we don't know of this port + return std::unique_ptr(); + } + + return std::make_unique( + port_name, *this, false, std::move(*version_entries)); + }); + }); + } + + FilesystemFromGitRegistryEntry::FilesystemFromGitRegistryEntry(StringView port_name, + const FilesystemFromGitRegistry& parent, + bool stale, + std::vector&& version_entries) + : port_name(port_name.data(), port_name.size()) + , parent(parent) + , stale(stale) + , last_loaded(std::move(version_entries)) + { + } + + ExpectedL> FilesystemFromGitRegistry::get_baseline_version(StringView port_name) const + { + return lookup_in_maybe_baseline(m_baseline.get([this, port_name]() -> ExpectedL { + // We delay baseline validation until here to give better error messages and suggestions + if (!is_git_commit_sha(m_baseline_identifier)) + { + auto& maybe_lock_entry = get_lock_entry(); + auto lock_entry = maybe_lock_entry.get(); + if (!lock_entry) + { + return maybe_lock_entry.error(); + } + + auto maybe_up_to_date = lock_entry->ensure_up_to_date(m_paths); + if (maybe_up_to_date) + { + return msg::format_error( + msgGitRegistryMustHaveBaseline, msg::url = m_repo, msg::commit_sha = lock_entry->commit_id()); + } + + return std::move(maybe_up_to_date).error(); + } + + auto path_to_baseline = Path(FileVersions) / FileBaselineDotJson; + auto maybe_contents = m_paths.git_show_from_remote_registry(m_baseline_identifier, path_to_baseline); + if (!maybe_contents) + { + auto& maybe_lock_entry = get_lock_entry(); + auto lock_entry = maybe_lock_entry.get(); + if (!lock_entry) + { + return maybe_lock_entry.error(); + } + + auto maybe_up_to_date = lock_entry->ensure_up_to_date(m_paths); + if (!maybe_up_to_date) + { + return std::move(maybe_up_to_date).error(); + } + + maybe_contents = m_paths.git_show_from_remote_registry(m_baseline_identifier, path_to_baseline); + } + + if (!maybe_contents) + { + msg::println(msgFetchingBaselineInfo, msg::package_name = m_repo); + auto maybe_err = m_paths.git_fetch(m_repo, m_baseline_identifier); + if (!maybe_err) + { + get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorCouldNotFindBaseline); + return msg::format_error(msgFailedToFetchRepo, msg::url = m_repo) + .append_raw('\n') + .append(maybe_err.error()); + } + + maybe_contents = m_paths.git_show_from_remote_registry(m_baseline_identifier, path_to_baseline); + } + + if (!maybe_contents) + { + get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorCouldNotFindBaseline); + return msg::format_error(msgCouldNotFindBaselineInCommit, + msg::url = m_repo, + msg::commit_sha = m_baseline_identifier, + msg::package_name = port_name) + .append_raw('\n') + .append_raw(maybe_contents.error()); + } + + auto contents = maybe_contents.get(); + return parse_baseline_versions(*contents, JsonIdDefault, path_to_baseline) + .map_error([&](LocalizedString&& error) { + get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorCouldNotFindBaseline); + return msg::format_error(msgErrorWhileFetchingBaseline, + msg::value = m_baseline_identifier, + msg::package_name = m_repo) + .append_raw('\n') + .append(error); + }); + }), + port_name); + } + + ExpectedL FilesystemFromGitRegistry::append_all_port_names(std::vector& out) const + { + auto maybe_versions_path = get_stale_versions_tree_path(); + if (auto versions_path = maybe_versions_path.get()) + { + return load_all_port_names_from_registry_versions(out, m_paths.get_filesystem(), versions_path->p); + } + + return std::move(maybe_versions_path).error(); + } + + ExpectedL FilesystemFromGitRegistry::try_append_all_port_names_no_network(std::vector&) const + { + // At this time we don't record enough information to know what the last fetch for a registry is, + // so we can't even return what the most recent answer was. + // + // This would be fixable if we recorded LockFile in the registries cache. + return false; + } + // } FilesystemFromGitRegistry::RegistryImplementation + // { GitRegistry::RegistryImplementation ExpectedL> GitRegistry::get_port_entry(StringView port_name) const { @@ -1113,6 +1445,89 @@ namespace } // } FilesystemRegistryEntry::RegistryEntry + // { FilesystemFromGitRegistryEntry::RegistryEntry + ExpectedL FilesystemFromGitRegistryEntry::ensure_not_stale() const + { + if (stale) + { + auto maybe_live_vdb = parent.get_versions_tree_path(); + auto live_vdb = maybe_live_vdb.get(); + if (!live_vdb) + { + return std::move(maybe_live_vdb).error(); + } + + auto maybe_maybe_version_entries = + load_git_versions_file(parent.m_paths.get_filesystem(), *live_vdb, port_name).entries; + auto maybe_version_entries = maybe_maybe_version_entries.get(); + if (!maybe_version_entries) + { + return std::move(maybe_maybe_version_entries).error(); + } + + auto version_entries = maybe_version_entries->get(); + if (!version_entries) + { + // Somehow the port existed in the stale version database but doesn't exist in the + // live one? + return msg::format_error(msgCouldNotFindVersionDatabaseFile, + msg::path = *live_vdb / relative_path_to_versions(port_name)); + } + + last_loaded.assign(std::move(*version_entries)); + stale = false; + } + + return Unit{}; + } + + ExpectedL> FilesystemFromGitRegistryEntry::get_port_versions() const + { + // Getting all versions that might exist must always be done with 'live' data + auto maybe_not_stale = ensure_not_stale(); + if (maybe_not_stale) + { + return View{last_loaded.port_versions()}; + } + + return std::move(maybe_not_stale).error(); + } + + ExpectedL FilesystemFromGitRegistryEntry::try_load_port(const Version& version) const + { + auto it = std::find(last_loaded.port_versions().begin(), last_loaded.port_versions().end(), version); + if (it == last_loaded.port_versions().end() && stale) + { + // didn't find the version, maybe a newer version database will have it + auto maybe_not_stale = ensure_not_stale(); + if (!maybe_not_stale) + { + return std::move(maybe_not_stale).error(); + } + + it = std::find(last_loaded.port_versions().begin(), last_loaded.port_versions().end(), version); + } + + if (it == last_loaded.port_versions().end()) + { + return format_version_git_entry_missing(port_name, version, last_loaded.port_versions()); + } + + const auto& git_tree = last_loaded.git_trees()[it - last_loaded.port_versions().begin()]; + return parent.m_paths.git_extract_tree_from_remote_registry(git_tree).then( + [this, &git_tree](Path&& p) -> ExpectedL { + return Paragraphs::try_load_port_required(parent.m_paths.get_filesystem(), + port_name, + PortLocation{ + p, + Strings::concat("git+", parent.m_repo, "@", git_tree), + }) + .maybe_scfl; + }); + } + + // } FilesystemFromGitRegistryEntry::RegistryEntry + // { GitRegistryEntry::RegistryEntry ExpectedL GitRegistryEntry::ensure_not_stale() const { From dd345c4bb807ab2200b4353d14daced27c02ea72 Mon Sep 17 00:00:00 2001 From: Vincent Le Garrec Date: Fri, 22 Nov 2024 16:12:32 +0100 Subject: [PATCH 4/7] Adapt FilesystemFromGitRegistry to make it work --- src/vcpkg/registries.cpp | 48 +++++++--------------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index b966b3d8c7..0ccc3c217f 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -472,8 +472,7 @@ namespace return std::move(maybe_up_to_date).error(); } - auto maybe_tree = m_paths.git_find_object_id_for_remote_registry_path(lock_entry->commit_id(), - FileVersions.to_string()); + auto maybe_tree = m_paths.git_find_object_id_for_remote_registry_path(lock_entry->commit_id(), ""); auto tree = maybe_tree.get(); if (!tree) { @@ -1046,50 +1045,19 @@ namespace // { FilesystemFromGitRegistry::RegistryImplementation ExpectedL> FilesystemFromGitRegistry::get_port_entry(StringView port_name) const { - auto maybe_stale_vtp = get_stale_versions_tree_path(); - auto stale_vtp = maybe_stale_vtp.get(); - if (!stale_vtp) - { - return std::move(maybe_stale_vtp).error(); - } - - { - // try to load using "stale" version database - auto maybe_maybe_version_entries = - load_git_versions_file(m_paths.get_filesystem(), stale_vtp->p, port_name).entries; - auto maybe_version_entries = maybe_maybe_version_entries.get(); - if (!maybe_version_entries) - { - return std::move(maybe_maybe_version_entries).error(); - } - - auto version_entries = maybe_version_entries->get(); - if (version_entries) - { - return std::make_unique( - port_name, *this, stale_vtp->stale, std::move(*version_entries)); - } - } - - if (!stale_vtp->stale) - { - // data is already live but we don't know of this port - return std::unique_ptr(); - } - return get_versions_tree_path().then([this, &port_name](const Path& live_vcb) { - return load_git_versions_file(m_paths.get_filesystem(), live_vcb, port_name) - .entries.then([this, &port_name](Optional>&& maybe_version_entries) - -> ExpectedL> { + return load_filesystem_versions_file( + m_paths.get_filesystem(), live_vcb / Path(FileVersions), port_name, live_vcb) + .then([&](Optional>&& maybe_version_entries) + -> ExpectedL> { auto version_entries = maybe_version_entries.get(); if (!version_entries) { - // data is already live but we don't know of this port - return std::unique_ptr(); + return std::unique_ptr{}; } - return std::make_unique( - port_name, *this, false, std::move(*version_entries)); + return std::make_unique( + m_paths.get_filesystem(), port_name, std::move(*version_entries)); }); }); } From 94959e7475ea2f799a69d9f6ece2d5a1d75e6663 Mon Sep 17 00:00:00 2001 From: Vincent Le Garrec Date: Fri, 22 Nov 2024 16:12:42 +0100 Subject: [PATCH 5/7] Remove unused FilesystemFromGitRegistryEntry --- src/vcpkg/registries.cpp | 121 --------------------------------------- 1 file changed, 121 deletions(-) diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index 0ccc3c217f..a63e7584ca 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -399,31 +399,6 @@ namespace DelayedInit> m_baseline; }; - struct FilesystemFromGitRegistry; - - struct FilesystemFromGitRegistryEntry final : RegistryEntry - { - FilesystemFromGitRegistryEntry(StringView port_name, - const FilesystemFromGitRegistry& parent, - bool stale, - std::vector&& version_entries); - - ExpectedL> get_port_versions() const override; - ExpectedL try_load_port(const Version& version) const override; - - private: - ExpectedL ensure_not_stale() const; - - std::string port_name; - - const FilesystemFromGitRegistry& parent; - - // Indicates whether port_versions and git_trees were filled in with stale (i.e. lock) data. - mutable bool stale; - - mutable PortVersionsGitTreesStructOfArrays last_loaded; - }; - struct FilesystemFromGitRegistry final : RegistryImplementation { FilesystemFromGitRegistry(const VcpkgPaths& paths, @@ -448,8 +423,6 @@ namespace ExpectedL> get_baseline_version(StringView) const override; private: - friend FilesystemFromGitRegistryEntry; - const ExpectedL& get_lock_entry() const { return m_lock_entry.get( @@ -1062,17 +1035,6 @@ namespace }); } - FilesystemFromGitRegistryEntry::FilesystemFromGitRegistryEntry(StringView port_name, - const FilesystemFromGitRegistry& parent, - bool stale, - std::vector&& version_entries) - : port_name(port_name.data(), port_name.size()) - , parent(parent) - , stale(stale) - , last_loaded(std::move(version_entries)) - { - } - ExpectedL> FilesystemFromGitRegistry::get_baseline_version(StringView port_name) const { return lookup_in_maybe_baseline(m_baseline.get([this, port_name]() -> ExpectedL { @@ -1413,89 +1375,6 @@ namespace } // } FilesystemRegistryEntry::RegistryEntry - // { FilesystemFromGitRegistryEntry::RegistryEntry - ExpectedL FilesystemFromGitRegistryEntry::ensure_not_stale() const - { - if (stale) - { - auto maybe_live_vdb = parent.get_versions_tree_path(); - auto live_vdb = maybe_live_vdb.get(); - if (!live_vdb) - { - return std::move(maybe_live_vdb).error(); - } - - auto maybe_maybe_version_entries = - load_git_versions_file(parent.m_paths.get_filesystem(), *live_vdb, port_name).entries; - auto maybe_version_entries = maybe_maybe_version_entries.get(); - if (!maybe_version_entries) - { - return std::move(maybe_maybe_version_entries).error(); - } - - auto version_entries = maybe_version_entries->get(); - if (!version_entries) - { - // Somehow the port existed in the stale version database but doesn't exist in the - // live one? - return msg::format_error(msgCouldNotFindVersionDatabaseFile, - msg::path = *live_vdb / relative_path_to_versions(port_name)); - } - - last_loaded.assign(std::move(*version_entries)); - stale = false; - } - - return Unit{}; - } - - ExpectedL> FilesystemFromGitRegistryEntry::get_port_versions() const - { - // Getting all versions that might exist must always be done with 'live' data - auto maybe_not_stale = ensure_not_stale(); - if (maybe_not_stale) - { - return View{last_loaded.port_versions()}; - } - - return std::move(maybe_not_stale).error(); - } - - ExpectedL FilesystemFromGitRegistryEntry::try_load_port(const Version& version) const - { - auto it = std::find(last_loaded.port_versions().begin(), last_loaded.port_versions().end(), version); - if (it == last_loaded.port_versions().end() && stale) - { - // didn't find the version, maybe a newer version database will have it - auto maybe_not_stale = ensure_not_stale(); - if (!maybe_not_stale) - { - return std::move(maybe_not_stale).error(); - } - - it = std::find(last_loaded.port_versions().begin(), last_loaded.port_versions().end(), version); - } - - if (it == last_loaded.port_versions().end()) - { - return format_version_git_entry_missing(port_name, version, last_loaded.port_versions()); - } - - const auto& git_tree = last_loaded.git_trees()[it - last_loaded.port_versions().begin()]; - return parent.m_paths.git_extract_tree_from_remote_registry(git_tree).then( - [this, &git_tree](Path&& p) -> ExpectedL { - return Paragraphs::try_load_port_required(parent.m_paths.get_filesystem(), - port_name, - PortLocation{ - p, - Strings::concat("git+", parent.m_repo, "@", git_tree), - }) - .maybe_scfl; - }); - } - - // } FilesystemFromGitRegistryEntry::RegistryEntry - // { GitRegistryEntry::RegistryEntry ExpectedL GitRegistryEntry::ensure_not_stale() const { From 67e0dc0bea299aab8220a98b544b74334d2a4cf6 Mon Sep 17 00:00:00 2001 From: Vincent Le Garrec Date: Fri, 22 Nov 2024 16:25:42 +0100 Subject: [PATCH 6/7] Use inheritance to avoid duplicated code --- src/vcpkg/registries.cpp | 243 ++------------------------------------- 1 file changed, 10 insertions(+), 233 deletions(-) diff --git a/src/vcpkg/registries.cpp b/src/vcpkg/registries.cpp index a63e7584ca..aa6cfbd821 100644 --- a/src/vcpkg/registries.cpp +++ b/src/vcpkg/registries.cpp @@ -261,13 +261,14 @@ namespace mutable PortVersionsGitTreesStructOfArrays last_loaded; }; - struct GitRegistry final : RegistryImplementation + struct GitRegistry : RegistryImplementation { GitRegistry(const VcpkgPaths& paths, std::string&& repo, std::string&& reference, std::string&& baseline) : m_paths(paths) , m_repo(std::move(repo)) , m_reference(std::move(reference)) , m_baseline_identifier(std::move(baseline)) + , m_remote_root_path(FileVersions.to_string()) { } @@ -281,7 +282,7 @@ namespace ExpectedL> get_baseline_version(StringView) const override; - private: + protected: friend GitRegistryEntry; const ExpectedL& get_lock_entry() const @@ -290,7 +291,7 @@ namespace [this]() { return m_paths.get_installed_lockfile().get_or_fetch(m_paths, m_repo, m_reference); }); } - const ExpectedL& get_versions_tree_path() const + virtual const ExpectedL& get_versions_tree_path() const { return m_versions_tree.get([this]() -> ExpectedL { auto& maybe_lock_entry = get_lock_entry(); @@ -306,8 +307,8 @@ namespace return std::move(maybe_up_to_date).error(); } - auto maybe_tree = m_paths.git_find_object_id_for_remote_registry_path(lock_entry->commit_id(), - FileVersions.to_string()); + auto maybe_tree = + m_paths.git_find_object_id_for_remote_registry_path(lock_entry->commit_id(), m_remote_root_path); auto tree = maybe_tree.get(); if (!tree) { @@ -397,144 +398,23 @@ namespace mutable Optional m_stale_versions_tree; DelayedInit> m_versions_tree; DelayedInit> m_baseline; + std::string m_remote_root_path; }; - struct FilesystemFromGitRegistry final : RegistryImplementation + struct FilesystemFromGitRegistry final : GitRegistry { FilesystemFromGitRegistry(const VcpkgPaths& paths, std::string&& repo, std::string&& reference, std::string&& baseline) - : m_paths(paths) - , m_repo(std::move(repo)) - , m_reference(std::move(reference)) - , m_baseline_identifier(std::move(baseline)) + : GitRegistry(paths, std::move(repo), std::move(reference), std::move(baseline)) { + m_remote_root_path = ""; } StringLiteral kind() const override { return "filesystem-from-git"; } ExpectedL> get_port_entry(StringView) const override; - - ExpectedL append_all_port_names(std::vector&) const override; - - ExpectedL try_append_all_port_names_no_network(std::vector& port_names) const override; - - ExpectedL> get_baseline_version(StringView) const override; - - private: - const ExpectedL& get_lock_entry() const - { - return m_lock_entry.get( - [this]() { return m_paths.get_installed_lockfile().get_or_fetch(m_paths, m_repo, m_reference); }); - } - - const ExpectedL& get_versions_tree_path() const - { - return m_versions_tree.get([this]() -> ExpectedL { - auto& maybe_lock_entry = get_lock_entry(); - auto lock_entry = maybe_lock_entry.get(); - if (!lock_entry) - { - return maybe_lock_entry.error(); - } - - auto maybe_up_to_date = lock_entry->ensure_up_to_date(m_paths); - if (!maybe_up_to_date) - { - return std::move(maybe_up_to_date).error(); - } - - auto maybe_tree = m_paths.git_find_object_id_for_remote_registry_path(lock_entry->commit_id(), ""); - auto tree = maybe_tree.get(); - if (!tree) - { - get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorNoVersionsAtCommit); - return msg::format_error(msgCouldNotFindGitTreeAtCommit, - msg::package_name = m_repo, - msg::commit_sha = lock_entry->commit_id()) - .append_raw('\n') - .append_raw(maybe_tree.error()); - } - - auto maybe_path = m_paths.git_extract_tree_from_remote_registry(*tree); - auto path = maybe_path.get(); - if (!path) - { - return msg::format_error(msgFailedToCheckoutRepo, msg::package_name = m_repo) - .append_raw('\n') - .append(maybe_path.error()); - } - - return std::move(*path); - }); - } - - struct VersionsTreePathResult - { - Path p; - bool stale; - }; - - ExpectedL get_unstale_stale_versions_tree_path() const - { - auto& maybe_versions_tree = get_versions_tree_path(); - if (auto versions_tree = maybe_versions_tree.get()) - { - return VersionsTreePathResult{*versions_tree, false}; - } - - return maybe_versions_tree.error(); - } - - ExpectedL get_stale_versions_tree_path() const - { - const auto& maybe_entry = get_lock_entry(); - auto entry = maybe_entry.get(); - if (!entry) - { - return maybe_entry.error(); - } - - if (!entry->stale()) - { - return get_unstale_stale_versions_tree_path(); - } - - if (!m_stale_versions_tree.has_value()) - { - auto maybe_tree = - m_paths.git_find_object_id_for_remote_registry_path(entry->commit_id(), FileVersions.to_string()); - auto tree = maybe_tree.get(); - if (!tree) - { - // This could be caused by git gc or otherwise -- fall back to full fetch - return get_unstale_stale_versions_tree_path(); - } - - auto maybe_path = m_paths.git_extract_tree_from_remote_registry(*tree); - auto path = maybe_path.get(); - if (!path) - { - // This could be caused by git gc or otherwise -- fall back to full fetch - return get_unstale_stale_versions_tree_path(); - } - - m_stale_versions_tree = std::move(*path); - } - - return VersionsTreePathResult{m_stale_versions_tree.value_or_exit(VCPKG_LINE_INFO), true}; - } - - const VcpkgPaths& m_paths; - - std::string m_repo; - std::string m_reference; - std::string m_baseline_identifier; - DelayedInit> m_lock_entry; - mutable Optional m_stale_versions_tree; - DelayedInit> m_versions_tree; - DelayedInit> m_baseline; }; struct BuiltinPortTreeRegistryEntry final : RegistryEntry @@ -1034,109 +914,6 @@ namespace }); }); } - - ExpectedL> FilesystemFromGitRegistry::get_baseline_version(StringView port_name) const - { - return lookup_in_maybe_baseline(m_baseline.get([this, port_name]() -> ExpectedL { - // We delay baseline validation until here to give better error messages and suggestions - if (!is_git_commit_sha(m_baseline_identifier)) - { - auto& maybe_lock_entry = get_lock_entry(); - auto lock_entry = maybe_lock_entry.get(); - if (!lock_entry) - { - return maybe_lock_entry.error(); - } - - auto maybe_up_to_date = lock_entry->ensure_up_to_date(m_paths); - if (maybe_up_to_date) - { - return msg::format_error( - msgGitRegistryMustHaveBaseline, msg::url = m_repo, msg::commit_sha = lock_entry->commit_id()); - } - - return std::move(maybe_up_to_date).error(); - } - - auto path_to_baseline = Path(FileVersions) / FileBaselineDotJson; - auto maybe_contents = m_paths.git_show_from_remote_registry(m_baseline_identifier, path_to_baseline); - if (!maybe_contents) - { - auto& maybe_lock_entry = get_lock_entry(); - auto lock_entry = maybe_lock_entry.get(); - if (!lock_entry) - { - return maybe_lock_entry.error(); - } - - auto maybe_up_to_date = lock_entry->ensure_up_to_date(m_paths); - if (!maybe_up_to_date) - { - return std::move(maybe_up_to_date).error(); - } - - maybe_contents = m_paths.git_show_from_remote_registry(m_baseline_identifier, path_to_baseline); - } - - if (!maybe_contents) - { - msg::println(msgFetchingBaselineInfo, msg::package_name = m_repo); - auto maybe_err = m_paths.git_fetch(m_repo, m_baseline_identifier); - if (!maybe_err) - { - get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorCouldNotFindBaseline); - return msg::format_error(msgFailedToFetchRepo, msg::url = m_repo) - .append_raw('\n') - .append(maybe_err.error()); - } - - maybe_contents = m_paths.git_show_from_remote_registry(m_baseline_identifier, path_to_baseline); - } - - if (!maybe_contents) - { - get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorCouldNotFindBaseline); - return msg::format_error(msgCouldNotFindBaselineInCommit, - msg::url = m_repo, - msg::commit_sha = m_baseline_identifier, - msg::package_name = port_name) - .append_raw('\n') - .append_raw(maybe_contents.error()); - } - - auto contents = maybe_contents.get(); - return parse_baseline_versions(*contents, JsonIdDefault, path_to_baseline) - .map_error([&](LocalizedString&& error) { - get_global_metrics_collector().track_define(DefineMetric::RegistriesErrorCouldNotFindBaseline); - return msg::format_error(msgErrorWhileFetchingBaseline, - msg::value = m_baseline_identifier, - msg::package_name = m_repo) - .append_raw('\n') - .append(error); - }); - }), - port_name); - } - - ExpectedL FilesystemFromGitRegistry::append_all_port_names(std::vector& out) const - { - auto maybe_versions_path = get_stale_versions_tree_path(); - if (auto versions_path = maybe_versions_path.get()) - { - return load_all_port_names_from_registry_versions(out, m_paths.get_filesystem(), versions_path->p); - } - - return std::move(maybe_versions_path).error(); - } - - ExpectedL FilesystemFromGitRegistry::try_append_all_port_names_no_network(std::vector&) const - { - // At this time we don't record enough information to know what the last fetch for a registry is, - // so we can't even return what the most recent answer was. - // - // This would be fixable if we recorded LockFile in the registries cache. - return false; - } // } FilesystemFromGitRegistry::RegistryImplementation // { GitRegistry::RegistryImplementation From 348d46e8476782113466fcc94e76d9b2048b351c Mon Sep 17 00:00:00 2001 From: Vincent Le Garrec Date: Sat, 23 Nov 2024 07:39:57 +0100 Subject: [PATCH 7/7] Fix workflow --- locales/messages.json | 1 + src/vcpkg-test/configmetadata.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/locales/messages.json b/locales/messages.json index d33ef52815..e8e7366006 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -14,6 +14,7 @@ "AFeature": "a feature", "AFeatureName": "a feature name", "AFilesystemRegistry": "a filesystem registry", + "AFilesystemRegistryFromGit": "a filesystem registry from git", "AGitObjectSha": "a git object SHA", "AGitReference": "a git reference (for example, a branch)", "AGitRegistry": "a git registry", diff --git a/src/vcpkg-test/configmetadata.cpp b/src/vcpkg-test/configmetadata.cpp index 57373bd03d..c317e1aa79 100644 --- a/src/vcpkg-test/configmetadata.cpp +++ b/src/vcpkg-test/configmetadata.cpp @@ -183,7 +183,7 @@ test: error: $ (a configuration object): The default registry cannot be an artif }] })json"; check_errors(raw_bad_kind, R"( -test: error: $.registries[0] (a registry): "kind" did not have an expected value: (expected one of: builtin, filesystem, git, artifact; found custom) +test: error: $.registries[0] (a registry): "kind" did not have an expected value: (expected one of: builtin, filesystem, filesystem-from-git, git, artifact; found custom) test: error: $.registries[0]: mismatched type: expected a registry )");