GitHub Dependency Graph API (#989)

* fix * fix * fix * fix * fix * fix * fix * fix * fix * fix
microsoft · Jun 20, 2023 · b9c33bb · b9c33bb
1 parent 8c254a5
commit b9c33bb
Show file tree

Hide file tree

Showing 9 changed files with 234 additions and 0 deletions.
diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h
@@ -5,6 +5,7 @@
 #include <vcpkg/base/fwd/messages.h>
 
 #include <vcpkg/base/expected.h>
+#include <vcpkg/base/json.h>
 #include <vcpkg/base/optional.h>
 #include <vcpkg/base/span.h>
 #include <vcpkg/base/stringview.h>
@@ -34,6 +35,10 @@ namespace vcpkg
     std::vector<int> download_files(const Filesystem& fs,
                                     View<std::pair<std::string, Path>> url_pairs,
                                     View<std::string> headers);
+
+    bool send_snapshot_to_api(const std::string& github_token,
+                              const std::string& github_repository,
+                              const Json::Object& snapshot);
     ExpectedL<int> put_file(const ReadOnlyFilesystem&,
                             StringView url,
                             const std::vector<std::string>& secrets,

diff --git a/include/vcpkg/commands.set-installed.h b/include/vcpkg/commands.set-installed.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <vcpkg/base/fwd/json.h>
 #include <vcpkg/base/fwd/optional.h>
 
 #include <vcpkg/fwd/binarycaching.h>
@@ -46,4 +47,7 @@ namespace vcpkg::Commands::SetInstalled
                           const VcpkgPaths& paths,
                           Triplet default_triplet,
                           Triplet host_triplet);
+
+    Optional<Json::Object> create_dependency_graph_snapshot(const VcpkgCmdArguments& args,
+                                                            const ActionPlan& action_plan);
 }
diff --git a/include/vcpkg/metrics.h b/include/vcpkg/metrics.h
@@ -84,8 +84,10 @@ namespace vcpkg
     enum class BoolMetric
     {
         DetectedContainer,
+        DependencyGraphSuccess,
         FeatureFlagBinaryCaching,
         FeatureFlagCompilerTracking,
+        FeatureFlagDependencyGraph,
         FeatureFlagManifests,
         FeatureFlagRegistries,
         FeatureFlagVersions,

diff --git a/include/vcpkg/vcpkgcmdarguments.h b/include/vcpkg/vcpkgcmdarguments.h
@@ -77,6 +77,7 @@ namespace vcpkg
         bool compiler_tracking;
         bool binary_caching;
         bool versions;
+        bool dependency_graph;
     };
 
     struct VcpkgCmdArguments
@@ -187,10 +188,22 @@ namespace vcpkg
         constexpr static StringLiteral ASSET_SOURCES_ENV = "X_VCPKG_ASSET_SOURCES";
         constexpr static StringLiteral ASSET_SOURCES_ARG = "asset-sources";
 
+        constexpr static StringLiteral GITHUB_RUN_ID_ENV = "GITHUB_RUN_ID";
+        Optional<std::string> github_run_id;
+        constexpr static StringLiteral GITHUB_TOKEN_ENV = "GITHUB_TOKEN";
+        Optional<std::string> github_token;
+        constexpr static StringLiteral GITHUB_JOB_ENV = "GITHUB_JOB";
+        Optional<std::string> github_job;
+        constexpr static StringLiteral GITHUB_WORKFLOW_ENV = "GITHUB_WORKFLOW";
+        Optional<std::string> github_workflow;
+
         // feature flags
         constexpr static StringLiteral FEATURE_FLAGS_ENV = "VCPKG_FEATURE_FLAGS";
         constexpr static StringLiteral FEATURE_FLAGS_ARG = "feature-flags";
 
+        constexpr static StringLiteral DEPENDENCY_GRAPH_FEATURE = "dependencygraph";
+        Optional<bool> dependency_graph_feature = nullopt;
+
         constexpr static StringLiteral FEATURE_PACKAGES_SWITCH = "featurepackages";
         Optional<bool> feature_packages = nullopt;
         constexpr static StringLiteral BINARY_CACHING_FEATURE = "binarycaching";
@@ -206,6 +219,7 @@ namespace vcpkg
 
         constexpr static StringLiteral RECURSIVE_DATA_ENV = "X_VCPKG_RECURSIVE_DATA";
 
+        bool dependency_graph_enabled() const { return dependency_graph_feature.value_or(false); }
         bool binary_caching_enabled() const { return binary_caching.value_or(true); }
         bool compiler_tracking_enabled() const { return compiler_tracking.value_or(true); }
         bool registries_enabled() const { return registries_feature.value_or(true); }
@@ -217,6 +231,7 @@ namespace vcpkg
             f.compiler_tracking = compiler_tracking_enabled();
             f.registries = registries_enabled();
             f.versions = versions_enabled();
+            f.dependency_graph = dependency_graph_enabled();
             return f;
         }
         const Optional<StringLiteral>& detected_ci_environment() const { return m_detected_ci_environment; }

diff --git a/src/vcpkg-test/dependencies.cpp b/src/vcpkg-test/dependencies.cpp
@@ -1,7 +1,11 @@
 #include <catch2/catch.hpp>
 
+#include <vcpkg/fwd/packagespec.h>
+
+#include <vcpkg/base/files.h>
 #include <vcpkg/base/graphs.h>
 
+#include <vcpkg/commands.set-installed.h>
 #include <vcpkg/dependencies.h>
 #include <vcpkg/portfileprovider.h>
 #include <vcpkg/sourceparagraph.h>
@@ -2364,3 +2368,60 @@ TEST_CASE ("formatting plan 1", "[dependencies]")
                   "    c:x64-osx -> 1 -- c\n"
                   "Additional packages (*) will be modified to complete this operation.\n");
 }
+
+TEST_CASE ("dependency graph API snapshot")
+{
+    MockVersionedPortfileProvider vp;
+    auto& scfl_a = vp.emplace("a", {"1", 0});
+    InstallPlanAction install_a(
+        {"a", Test::X64_WINDOWS}, scfl_a, RequestType::AUTO_SELECTED, Test::X64_ANDROID, {}, {});
+
+    ActionPlan plan;
+    plan.install_actions.push_back(std::move(install_a));
+    std::map<std::string, std::string, std::less<>> envmap = {
+        {VcpkgCmdArguments::GITHUB_JOB_ENV.to_string(), "123"},
+        {VcpkgCmdArguments::GITHUB_RUN_ID_ENV.to_string(), "123"},
+        {VcpkgCmdArguments::GITHUB_REF_ENV.to_string(), "refs/heads/main"},
+        {VcpkgCmdArguments::GITHUB_REPOSITORY_ENV.to_string(), "owner/repo"},
+        {VcpkgCmdArguments::GITHUB_SHA_ENV.to_string(), "abc123"},
+        {VcpkgCmdArguments::GITHUB_TOKEN_ENV.to_string(), "abc"},
+        {VcpkgCmdArguments::GITHUB_WORKFLOW_ENV.to_string(), "test"},
+    };
+    auto v = VcpkgCmdArguments::create_from_arg_sequence(nullptr, nullptr);
+    v.imbue_from_fake_environment(envmap);
+    auto s = vcpkg::Commands::SetInstalled::create_dependency_graph_snapshot(v, plan);
+
+    CHECK(s.has_value());
+    auto obj = *s.get();
+    auto version = obj.get("version")->integer(VCPKG_LINE_INFO);
+    auto job = obj.get("job")->object(VCPKG_LINE_INFO);
+    auto id = job.get("id")->string(VCPKG_LINE_INFO);
+    auto correlator = job.get("correlator")->string(VCPKG_LINE_INFO);
+    auto sha = obj.get("sha")->string(VCPKG_LINE_INFO);
+    auto ref = obj.get("ref")->string(VCPKG_LINE_INFO);
+    auto detector = obj.get("detector")->object(VCPKG_LINE_INFO);
+    auto name = detector.get("name")->string(VCPKG_LINE_INFO);
+    auto detector_version = detector.get("version")->string(VCPKG_LINE_INFO);
+    auto url = detector.get("url")->string(VCPKG_LINE_INFO);
+    auto manifests = obj.get("manifests")->object(VCPKG_LINE_INFO);
+    auto manifest1 = manifests.get("vcpkg.json")->object(VCPKG_LINE_INFO);
+    auto name1 = manifest1.get("name")->string(VCPKG_LINE_INFO);
+    auto resolved1 = manifest1.get("resolved")->object(VCPKG_LINE_INFO);
+    auto dependency_a = resolved1.get("pkg:github/vcpkg/a@1")->object(VCPKG_LINE_INFO);
+    auto package_url_a = dependency_a.get("package_url")->string(VCPKG_LINE_INFO);
+    auto relationship_a = dependency_a.get("relationship")->string(VCPKG_LINE_INFO);
+    auto dependencies_a = dependency_a.get("dependencies")->array(VCPKG_LINE_INFO);
+
+    CHECK(static_cast<int>(version) == 0);
+    CHECK(id == "123");
+    CHECK(correlator == "test-123");
+    CHECK(sha == "abc123");
+    CHECK(ref == "refs/heads/main");
+    CHECK(name == "vcpkg");
+    CHECK(detector_version == "1.0.0");
+    CHECK(url == "https://github.com/microsoft/vcpkg");
+    CHECK(name1 == "vcpkg.json");
+    CHECK(package_url_a == "pkg:github/vcpkg/a@1");
+    CHECK(relationship_a == "direct");
+    CHECK(dependencies_a.size() == 0);
+}
diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp
@@ -3,6 +3,7 @@
 #include <vcpkg/base/downloads.h>
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/hash.h>
+#include <vcpkg/base/json.h>
 #include <vcpkg/base/message_sinks.h>
 #include <vcpkg/base/parse.h>
 #include <vcpkg/base/strings.h>
@@ -12,6 +13,8 @@
 #include <vcpkg/base/system.proxy.h>
 #include <vcpkg/base/util.h>
 
+#include <vcpkg/metrics.h>
+
 namespace vcpkg
 {
     static std::string replace_secrets(std::string input, View<std::string> secrets)
@@ -517,6 +520,44 @@ namespace vcpkg
         return ret;
     }
 
+    bool send_snapshot_to_api(const std::string& github_token,
+                              const std::string& github_repository,
+                              const Json::Object& snapshot)
+    {
+        static constexpr StringLiteral guid_marker = "fcfad8a3-bb68-4a54-ad00-dab1ff671ed2";
+
+        Command cmd;
+        cmd.string_arg("curl");
+        cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}");
+        cmd.string_arg("-X").string_arg("POST");
+        cmd.string_arg("-H").string_arg("Accept: application/vnd.github+json");
+
+        std::string res = "Authorization: Bearer " + github_token;
+        cmd.string_arg("-H").string_arg(res);
+        cmd.string_arg("-H").string_arg("X-GitHub-Api-Version: 2022-11-28");
+        cmd.string_arg(
+            Strings::concat("https://github.com/gitapi/repos/", github_repository, "/dependency-graph/snapshots"));
+        cmd.string_arg("-d").string_arg(Json::stringify(snapshot));
+        int code = 0;
+        auto result = cmd_execute_and_stream_lines(cmd, [&code](StringView line) {
+            if (Strings::starts_with(line, guid_marker))
+            {
+                code = std::strtol(line.data() + guid_marker.size(), nullptr, 10);
+            }
+            else
+            {
+                Debug::println(line);
+            }
+        });
+
+        auto r = result.get();
+        if (r && *r == 0 && code >= 200 && code < 300)
+        {
+            return true;
+        }
+        return false;
+    }
+
     ExpectedL<int> put_file(const ReadOnlyFilesystem&,
                             StringView url,
                             const std::vector<std::string>& secrets,

diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp
@@ -1,3 +1,8 @@
+#include <vcpkg/base/downloads.h>
+#include <vcpkg/base/json.h>
+#include <vcpkg/base/system.debug.h>
+#include <vcpkg/base/system.h>
+
 #include <vcpkg/binarycaching.h>
 #include <vcpkg/cmakevars.h>
 #include <vcpkg/commands.help.h>
@@ -42,6 +47,78 @@ namespace vcpkg::Commands::SetInstalled
         nullptr,
     };
 
+    Optional<Json::Object> create_dependency_graph_snapshot(const VcpkgCmdArguments& args,
+                                                            const ActionPlan& action_plan)
+    {
+        if (args.github_ref.has_value() && args.github_sha.has_value() && args.github_job.has_value() &&
+            args.github_workflow.has_value() && args.github_run_id.has_value())
+        {
+            Json::Object detector;
+            detector.insert("name", Json::Value::string("vcpkg"));
+            detector.insert("url", Json::Value::string("https://github.com/microsoft/vcpkg"));
+            detector.insert("version", Json::Value::string("1.0.0"));
+
+            Json::Object job;
+            job.insert("id", Json::Value::string(*args.github_run_id.get()));
+            job.insert("correlator", Json::Value::string(*args.github_workflow.get() + "-" + *args.github_job.get()));
+
+            Json::Object snapshot;
+            snapshot.insert("job", job);
+            snapshot.insert("version", Json::Value::integer(0));
+            snapshot.insert("sha", Json::Value::string(*args.github_sha.get()));
+            snapshot.insert("ref", Json::Value::string(*args.github_ref.get()));
+            snapshot.insert("scanned", Json::Value::string(CTime::now_string()));
+            snapshot.insert("detector", detector);
+
+            Json::Object manifest;
+            manifest.insert("name", "vcpkg.json");
+
+            std::unordered_map<std::string, std::string> map;
+            for (auto&& action : action_plan.install_actions)
+            {
+                if (!action.source_control_file_and_location.has_value())
+                {
+                    return nullopt;
+                }
+                const auto& scf = *action.source_control_file_and_location.get();
+                auto version = scf.to_version().to_string();
+                auto pkg_url = Strings::concat("pkg:github/vcpkg/", action.spec.name(), "@", version);
+                map.insert({action.spec.to_string(), pkg_url});
+            }
+
+            Json::Object resolved;
+            for (auto&& action : action_plan.install_actions)
+            {
+                Json::Object resolved_item;
+                if (map.find(action.spec.to_string()) != map.end())
+                {
+                    auto pkg_url = map.at(action.spec.to_string());
+                    resolved_item.insert("package_url", pkg_url);
+                    resolved_item.insert("relationship", Json::Value::string("direct"));
+                    Json::Array deps_list;
+                    for (auto&& dep : action.package_dependencies)
+                    {
+                        if (map.find(dep.to_string()) != map.end())
+                        {
+                            auto dep_pkg_url = map.at(dep.to_string());
+                            deps_list.push_back(dep_pkg_url);
+                        }
+                    }
+                    resolved_item.insert("dependencies", deps_list);
+                    resolved.insert(pkg_url, resolved_item);
+                }
+            }
+            manifest.insert("resolved", resolved);
+            Json::Object manifests;
+            manifests.insert("vcpkg.json", manifest);
+            snapshot.insert("manifests", manifests);
+
+            Debug::print(Json::stringify(snapshot));
+            return snapshot;
+        }
+        return nullopt;
+    }
+
     std::set<PackageSpec> adjust_action_plan_to_status_db(ActionPlan& action_plan, const StatusParagraphs& status_db)
     {
         std::set<std::string> all_abis;
@@ -109,6 +186,17 @@ namespace vcpkg::Commands::SetInstalled
             }
         }
 
+        if (paths.manifest_mode_enabled() && paths.get_feature_flags().dependency_graph)
+        {
+            auto snapshot = create_dependency_graph_snapshot(args, action_plan);
+            bool s = false;
+            if (snapshot.has_value() && args.github_token.has_value() && args.github_repository.has_value())
+            {
+                s = send_snapshot_to_api(*args.github_token.get(), *args.github_repository.get(), *snapshot.get());
+            }
+            get_global_metrics_collector().track_bool(BoolMetric::DependencyGraphSuccess, s);
+        }
+
         // currently (or once) installed specifications
         auto status_db = database_load_check(fs, paths.installed());
         adjust_action_plan_to_status_db(action_plan, status_db);

diff --git a/src/vcpkg/metrics.cpp b/src/vcpkg/metrics.cpp
@@ -148,8 +148,10 @@ namespace vcpkg
 
     const constexpr std::array<BoolMetricEntry, static_cast<size_t>(BoolMetric::COUNT)> all_bool_metrics{{
         {BoolMetric::DetectedContainer, "detected_container"},
+        {BoolMetric::DependencyGraphSuccess, "dependency-graph-success"},
         {BoolMetric::FeatureFlagBinaryCaching, "feature-flag-binarycaching"},
         {BoolMetric::FeatureFlagCompilerTracking, "feature-flag-compilertracking"},
+        {BoolMetric::FeatureFlagDependencyGraph, "feature-flag-dependency-graph"},
         {BoolMetric::FeatureFlagManifests, "feature-flag-manifests"},
         {BoolMetric::FeatureFlagRegistries, "feature-flag-registries"},
         {BoolMetric::FeatureFlagVersions, "feature-flag-versions"},

diff --git a/src/vcpkg/vcpkgcmdarguments.cpp b/src/vcpkg/vcpkgcmdarguments.cpp
@@ -222,6 +222,7 @@ namespace vcpkg
             {VcpkgCmdArguments::COMPILER_TRACKING_FEATURE, args.compiler_tracking},
             {VcpkgCmdArguments::REGISTRIES_FEATURE, args.registries_feature},
             {VcpkgCmdArguments::VERSIONS_FEATURE, args.versions_feature},
+            {VcpkgCmdArguments::DEPENDENCY_GRAPH_FEATURE, args.dependency_graph_feature},
         };
 
         for (const auto& desc : flag_descriptions)
@@ -493,6 +494,10 @@ namespace vcpkg
         from_env(get_env, GITHUB_SERVER_URL_ENV, github_server_url);
         from_env(get_env, GITHUB_REF_ENV, github_ref);
         from_env(get_env, GITHUB_SHA_ENV, github_sha);
+        from_env(get_env, GITHUB_JOB_ENV, github_job);
+        from_env(get_env, GITHUB_RUN_ID_ENV, github_run_id);
+        from_env(get_env, GITHUB_TOKEN_ENV, github_token);
+        from_env(get_env, GITHUB_WORKFLOW_ENV, github_workflow);
 
         // detect whether we are running in a CI environment
         for (auto&& ci_env_var : KNOWN_CI_VARIABLES)
@@ -657,6 +662,7 @@ namespace vcpkg
             {COMPILER_TRACKING_FEATURE, compiler_tracking},
             {REGISTRIES_FEATURE, registries_feature},
             {VERSIONS_FEATURE, versions_feature},
+            {DEPENDENCY_GRAPH_FEATURE, dependency_graph_feature},
         };
 
         for (const auto& flag : flags)
@@ -677,6 +683,7 @@ namespace vcpkg
         MetricsSubmission submission;
         submission.track_bool(BoolMetric::FeatureFlagBinaryCaching, binary_caching_enabled());
         submission.track_bool(BoolMetric::FeatureFlagCompilerTracking, compiler_tracking_enabled());
+        submission.track_bool(BoolMetric::FeatureFlagDependencyGraph, dependency_graph_enabled());
         submission.track_bool(BoolMetric::FeatureFlagRegistries, registries_enabled());
         submission.track_bool(BoolMetric::FeatureFlagVersions, versions_enabled());
         get_global_metrics_collector().track_submission(std::move(submission));
@@ -758,10 +765,19 @@ namespace vcpkg
     constexpr StringLiteral VcpkgCmdArguments::ASSET_SOURCES_ENV;
     constexpr StringLiteral VcpkgCmdArguments::ASSET_SOURCES_ARG;
 
+    constexpr StringLiteral VcpkgCmdArguments::GITHUB_JOB_ENV;
+    constexpr StringLiteral VcpkgCmdArguments::GITHUB_RUN_ID_ENV;
+    constexpr StringLiteral VcpkgCmdArguments::GITHUB_REPOSITORY_ENV;
+    constexpr StringLiteral VcpkgCmdArguments::GITHUB_REF_ENV;
+    constexpr StringLiteral VcpkgCmdArguments::GITHUB_SHA_ENV;
+    constexpr StringLiteral VcpkgCmdArguments::GITHUB_TOKEN_ENV;
+    constexpr StringLiteral VcpkgCmdArguments::GITHUB_WORKFLOW_ENV;
+
     constexpr StringLiteral VcpkgCmdArguments::FEATURE_FLAGS_ENV;
     constexpr StringLiteral VcpkgCmdArguments::FEATURE_FLAGS_ARG;
 
     constexpr StringLiteral VcpkgCmdArguments::FEATURE_PACKAGES_SWITCH;
+    constexpr StringLiteral VcpkgCmdArguments::DEPENDENCY_GRAPH_FEATURE;
     constexpr StringLiteral VcpkgCmdArguments::BINARY_CACHING_FEATURE;
     constexpr StringLiteral VcpkgCmdArguments::BINARY_CACHING_SWITCH;
     constexpr StringLiteral VcpkgCmdArguments::COMPILER_TRACKING_FEATURE;