Refactor Binary Caching

include/vcpkg-test/util.h

@@ -4,11 +4,14 @@
 
 #include <catch2/catch.hpp>
 
+#include <vcpkg/base/fwd/files.h>
+
+#include <vcpkg/fwd/tools.h>
+
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/format.h>
 #include <vcpkg/base/messages.h>
 #include <vcpkg/base/pragmas.h>
-#include <vcpkg/base/sortedvector.h>
 #include <vcpkg/base/strings.h>
 
 #include <vcpkg/packagespec.h>

include/vcpkg/archives.h

@@ -1,12 +1,15 @@
 #pragma once
 
+#include <vcpkg/base/fwd/expected.h>
 #include <vcpkg/base/fwd/files.h>
+#include <vcpkg/base/fwd/message_sinks.h>
 #include <vcpkg/base/fwd/span.h>
 #include <vcpkg/base/fwd/system.process.h>
 
+#include <vcpkg/fwd/tools.h>
 #include <vcpkg/fwd/vcpkgpaths.h>
 
-#include <vcpkg/tools.h>
+#include <vcpkg/base/path.h>
 
 namespace vcpkg
 {
@@ -27,14 +30,26 @@ namespace vcpkg
         Filesystem& fs, const ToolCache& tools, MessageSink& status_sink, const Path& archive, const Path& to_path);
 #endif
 
-    // Compress the source directory into the destination file.
-    ExpectedL<Unit> compress_directory_to_zip(
-        Filesystem& fs, const ToolCache& tools, MessageSink& status_sink, const Path& source, const Path& destination);
+    struct ZipTool
+    {
+#if defined _WIN32
+        ZipTool(RemoveFilesystem& fs, const ToolCache& tools, MessageSink& status_sink);
+#else
+        ZipTool(RemoveFilesystem& fs);
+#endif
+
+    private:
+        RemoveFilesystem* fs;
+#if defined _WIN32
+        Path seven_zip;
+#endif
+
+    public:
+        // Compress the source directory into the destination file.
+        ExpectedL<Unit> compress_directory_to_zip(const Path& source, const Path& destination) const;
 
-    Command decompress_zip_archive_cmd(const ToolCache& tools,
-                                       MessageSink& status_sink,
-                                       const Path& dst,
-                                       const Path& archive_path);
+        Command decompress_zip_archive_cmd(const Path& dst, const Path& archive_path) const;
+    };
 
     std::vector<ExpectedL<Unit>> decompress_in_parallel(View<Command> jobs);
 }

include/vcpkg/base/files.h

@@ -106,10 +106,23 @@ namespace vcpkg
         virtual ExpectedL<std::vector<std::string>> read_lines(const Path& file_path) const = 0;
 
     protected:
-        ~ILineReader();
+        ~ILineReader() = default;
     };
 
-    struct Filesystem : ILineReader
+    struct RemoveFilesystem
+    {
+        virtual bool remove(const Path& target, std::error_code& ec) = 0;
+        bool remove(const Path& target, LineInfo li);
+
+        virtual void remove_all(const Path& base, std::error_code& ec, Path& failure_point) = 0;
+        void remove_all(const Path& base, std::error_code& ec);
+        void remove_all(const Path& base, LineInfo li);
+
+    protected:
+        ~RemoveFilesystem() = default;
+    };
+
+    struct Filesystem : ILineReader, RemoveFilesystem
     {
         virtual std::string read_contents(const Path& file_path, std::error_code& ec) const = 0;
         std::string read_contents(const Path& file_path, LineInfo li) const;
@@ -164,13 +177,6 @@ namespace vcpkg
                                     StringLiteral temp_suffix,
                                     std::error_code& ec) = 0;
 
-        virtual bool remove(const Path& target, std::error_code& ec) = 0;
-        bool remove(const Path& target, LineInfo li);
-
-        virtual void remove_all(const Path& base, std::error_code& ec, Path& failure_point) = 0;
-        void remove_all(const Path& base, std::error_code& ec);
-        void remove_all(const Path& base, LineInfo li);
-
         void remove_all_inside(const Path& base, std::error_code& ec, Path& failure_point);
         void remove_all_inside(const Path& base, std::error_code& ec);
         void remove_all_inside(const Path& base, LineInfo li);

include/vcpkg/base/fwd/files.h

@@ -48,6 +48,7 @@ namespace vcpkg
     struct IExclusiveFileLock;
     struct ILineReader;
     struct FileContents;
+    struct RemoveFilesystem;
     struct Filesystem;
     struct NotExtensionCaseSensitive;
     struct NotExtensionCaseInsensitive;

include/vcpkg/base/message-data.inc.h

@@ -2282,7 +2282,10 @@ DECLARE_MESSAGE(SpecifyTargetArch,
                 "Specify the target architecture triplet. See 'vcpkg help triplet'.\n(default: '{env_var}')")
 DECLARE_MESSAGE(StartCodeUnitInContinue, (), "", "found start code unit in continue position")
 DECLARE_MESSAGE(StoredBinaryCache, (msg::path), "", "Stored binary cache: \"{path}\"")
-DECLARE_MESSAGE(StoredBinariesToDestinations, (msg::count), "", "Stored binaries in {count} destinations.")
+DECLARE_MESSAGE(StoredBinariesToDestinations,
+                (msg::count, msg::elapsed),
+                "",
+                "Stored binaries in {count} destinations in {elapsed}.")
 DECLARE_MESSAGE(StoreOptionMissingSha, (), "", "--store option is invalid without a sha512")
 DECLARE_MESSAGE(SuccessfulyExported, (msg::package_name, msg::path), "", "Exported {package_name} to {path}")
 DECLARE_MESSAGE(SuggestGitPull, (), "", "The result may be outdated. Run `git pull` to get the latest results.")

include/vcpkg/binarycaching.h

@@ -4,12 +4,14 @@
 
 #include <vcpkg/fwd/binarycaching.h>
 #include <vcpkg/fwd/dependencies.h>
+#include <vcpkg/fwd/tools.h>
 #include <vcpkg/fwd/vcpkgpaths.h>
 
 #include <vcpkg/base/downloads.h>
 #include <vcpkg/base/expected.h>
 #include <vcpkg/base/path.h>
 
+#include <vcpkg/archives.h>
 #include <vcpkg/packagespec.h>
 
 #include <iterator>
@@ -22,83 +24,103 @@ namespace vcpkg
 {
     struct CacheStatus
     {
-        bool should_attempt_precheck(const IBinaryProvider* sender) const noexcept;
-        bool should_attempt_restore(const IBinaryProvider* sender) const noexcept;
+        bool should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept;
+        bool should_attempt_restore(const IReadBinaryProvider* sender) const noexcept;
 
-        bool is_unavailable(size_t total_providers) const noexcept;
-        const IBinaryProvider* get_available_provider() const noexcept;
+        bool is_unavailable(const IReadBinaryProvider* sender) const noexcept;
+        const IReadBinaryProvider* get_available_provider() const noexcept;
         bool is_restored() const noexcept;
 
-        void mark_unavailable(const IBinaryProvider* sender);
-        void mark_available(const IBinaryProvider* sender) noexcept;
+        void mark_unavailable(const IReadBinaryProvider* sender);
+        void mark_available(const IReadBinaryProvider* sender) noexcept;
         void mark_restored() noexcept;
 
     private:
         CacheStatusState m_status = CacheStatusState::unknown;
 
         // The set of providers who know they do not have the associated cache entry.
         // Flat vector set because N is tiny.
-        std::vector<const IBinaryProvider*> m_known_unavailable_providers; // meaningful iff m_status == unknown
+        std::vector<const IReadBinaryProvider*> m_known_unavailable_providers;
 
         // The provider who affirmatively has the associated cache entry.
-        const IBinaryProvider* m_available_provider = nullptr; // meaningful iff m_status == available
+        const IReadBinaryProvider* m_available_provider = nullptr; // meaningful iff m_status == available
     };
 
-    struct BinaryPackageInformation
+    struct BinaryPackageReadInfo
     {
-        explicit BinaryPackageInformation(const InstallPlanAction& action, Optional<std::string> nuspec);
+        explicit BinaryPackageReadInfo(const InstallPlanAction& action);
         std::string package_abi;
         PackageSpec spec;
         std::string raw_version;
-        // only filled if BinaryCache has a provider that returns true for needs_nuspec_data()
-        Optional<std::string> nuspec;
+        Path package_dir;
     };
 
-    struct BinaryProviderPushRequest
+    struct BinaryPackageWriteInfo : BinaryPackageReadInfo
     {
-        BinaryPackageInformation info;
-        Path package_dir;
+        using BinaryPackageReadInfo::BinaryPackageReadInfo;
+
+        // Filled if BinaryCache has a provider that returns true for needs_nuspec_data()
+        Optional<std::string> nuspec;
+        // Filled if BinaryCache has a provider that returns true for needs_zip_file()
+        // Note: this can be empty if an error occurred while compressing.
+        Optional<Path> zip_path;
     };
 
-    struct IBinaryProvider
+    struct IWriteBinaryProvider
     {
-        virtual ~IBinaryProvider() = default;
-
-        /// Attempts to restore the package referenced by `action` into the packages directory.
-        /// Prerequisite: action has a package_abi()
-        virtual RestoreResult try_restore(const InstallPlanAction& action) const = 0;
+        virtual ~IWriteBinaryProvider() = default;
 
         /// Called upon a successful build of `action` to store those contents in the binary cache.
-        /// Prerequisite: action has a package_abi()
         /// returns the number of successful uploads
-        virtual size_t push_success(const BinaryProviderPushRequest& request, MessageSink& msg_sink) = 0;
+        virtual size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) = 0;
 
-        /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for
-        /// executing `actions`.
-        /// `cache_status` is a vector with the same number of entries of actions, where each index corresponds
-        /// to the action at the same index in `actions`. The provider must mark the cache status as appropriate.
-        /// Note: `actions` *might not* have package ABIs (for example, a --head package)!
-        /// Prerequisite: if `actions[i]` has no package ABI, `cache_status[i]` is nullptr.
-        virtual void prefetch(View<InstallPlanAction> actions, View<CacheStatus*> cache_status) const = 0;
+        virtual bool needs_nuspec_data() const = 0;
+        virtual bool needs_zip_file() const = 0;
+    };
 
-        /// Checks whether the `actions` are present in the cache, without restoring them. Used by CI to determine
-        /// missing packages.
-        /// `cache_status` is a view with the same number of entries of actions, where each index corresponds
-        /// to the action at the same index in `actions`. The provider must mark the cache status as appropriate.
-        /// Prerequisite: `actions` have package ABIs.
-        virtual void precheck(View<InstallPlanAction> actions, View<CacheStatus*> cache_status) const = 0;
+    struct IReadBinaryProvider
+    {
+        virtual ~IReadBinaryProvider() = default;
 
-        virtual bool needs_nuspec_data() const { return false; }
+        /// Attempts to restore the package referenced by `action` into the packages directory.
+        ///
+        /// Prerequisite: action has a package_abi()
+        virtual RestoreResult try_restore(const InstallPlanAction& action) const = 0;
+
+        /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for executing
+        /// `actions`.
+        ///
+        /// IBinaryProvider should set out_status[i] to RestoreResult::restored for each fetched package.
+        ///
+        /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size()
+        virtual void prefetch(View<const InstallPlanAction*> actions, Span<RestoreResult> out_status) const = 0;
+
+        /// Checks whether the `actions` are present in the cache, without restoring them.
+        ///
+        /// Used by CI to determine missing packages. For each `i`, out_status[i] should be set to
+        /// CacheAvailability::available or CacheAvailability::unavailable
+        ///
+        /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size()
+        virtual void precheck(View<const InstallPlanAction*> actions, Span<CacheAvailability> out_status) const = 0;
+
+        /// <returns>A user-visible vendor string identifying this provider</returns>
+        virtual StringView vendor() const = 0;
     };
 
     struct UrlTemplate
     {
         std::string url_template;
-        std::vector<std::string> headers_for_put;
-        std::vector<std::string> headers_for_get;
+        std::vector<std::string> headers;
 
         LocalizedString valid() const;
-        std::string instantiate_variables(const BinaryPackageInformation& info) const;
+        std::string instantiate_variables(const BinaryPackageReadInfo& info) const;
+    };
+
+    struct NuGetRepoInfo
+    {
+        std::string repo;
+        std::string branch;
+        std::string commit;
     };
 
     struct BinaryConfigParserState
@@ -135,28 +157,34 @@ namespace vcpkg
 
         std::vector<std::string> secrets;
 
+        // These are filled in after construction by reading from args and environment
+        std::string nuget_prefix;
+        bool use_nuget_cache = false;
+        NuGetRepoInfo nuget_repo_info;
+
         void clear();
     };
 
-    ExpectedL<BinaryConfigParserState> create_binary_providers_from_configs_pure(const std::string& env_string,
-                                                                                 View<std::string> args);
-    ExpectedL<std::vector<std::unique_ptr<IBinaryProvider>>> create_binary_providers_from_configs(
-        const VcpkgPaths& paths, View<std::string> args);
+    ExpectedL<BinaryConfigParserState> parse_binary_provider_configs(const std::string& env_string,
+                                                                     View<std::string> args);
 
-    struct BinaryCache
+    struct BinaryProviders
     {
-        BinaryCache(Filesystem& filesystem);
-        explicit BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths);
+        std::vector<std::unique_ptr<IReadBinaryProvider>> read;
+        std::vector<std::unique_ptr<IWriteBinaryProvider>> write;
+        std::string nuget_prefix;
+        NuGetRepoInfo nuget_repo;
+    };
 
-        void install_providers(std::vector<std::unique_ptr<IBinaryProvider>>&& providers);
-        void install_providers_for(const VcpkgCmdArguments& args, const VcpkgPaths& paths);
+    struct ReadOnlyBinaryCache
+    {
+        ReadOnlyBinaryCache() = default;
+        ReadOnlyBinaryCache(BinaryProviders&& providers);
+        ReadOnlyBinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info);
 
         /// Attempts to restore the package referenced by `action` into the packages directory.
         RestoreResult try_restore(const InstallPlanAction& action);
 
-        /// Called upon a successful build of `action` to store those contents in the binary cache.
-        void push_success(const InstallPlanAction& action, Path package_dir);
-
         /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for
         /// executing `actions`.
         void prefetch(View<InstallPlanAction> actions);
@@ -166,16 +194,32 @@ namespace vcpkg
         /// Returns a vector where each index corresponds to the matching index in `actions`.
         std::vector<CacheAvailability> precheck(View<InstallPlanAction> actions);
 
-    private:
+    protected:
+        BinaryProviders m_config;
+
         std::unordered_map<std::string, CacheStatus> m_status;
-        std::vector<std::unique_ptr<IBinaryProvider>> m_providers;
-        bool needs_nuspec_data = false;
-        Filesystem& filesystem;
+    };
+
+    struct BinaryCache : ReadOnlyBinaryCache
+    {
+        BinaryCache(RemoveFilesystem& fs);
+        BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info);
+        BinaryCache(BinaryProviders&& providers, RemoveFilesystem& fs, const ToolCache& tools);
+        ~BinaryCache();
+
+        /// Called upon a successful build of `action` to store those contents in the binary cache.
+        void push_success(const InstallPlanAction& action);
+
+    private:
+        RemoveFilesystem& m_fs;
+        Optional<ZipTool> m_zip_tool;
+        bool m_needs_nuspec_data = false;
+        bool m_needs_zip_file = false;
     };
 
     ExpectedL<DownloadManagerConfig> parse_download_configuration(const Optional<std::string>& arg);
 
-    std::string generate_nuget_packages_config(const ActionPlan& action);
+    std::string generate_nuget_packages_config(const ActionPlan& action, StringView prefix);
 
     LocalizedString format_help_topic_asset_caching();
     LocalizedString format_help_topic_binary_caching();