diff --git a/contrib/git-changes-action/README.md b/contrib/git-changes-action/README.md index 7225aa94b3..85b8e1ffb9 100644 --- a/contrib/git-changes-action/README.md +++ b/contrib/git-changes-action/README.md @@ -1,6 +1,6 @@ # Git Changes Action -This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request, along with any dependent modules. This can be useful for automating build, test, or deployment workflows that involve Go projects. +This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request, and if desired, along with any dependent modules. This can be useful for automating build, test, or deployment workflows that involve Go projects. [![Go Reference](https://pkg.go.dev/badge/github.com/synapsecns/sanguine/contrib/git-changes-action.svg)](https://pkg.go.dev/github.com/synapsecns/sanguine/contrib/git-changes-action) [![Go Report Card](https://goreportcard.com/badge/github.com/synapsecns/sanguine/contrib/git-changes-action)](https://goreportcard.com/report/github.com/synapsecns/sanguine/contrib/git-changes-action) @@ -9,9 +9,7 @@ This GitHub Action exports a variable that contains the list of Go modules chang 1. To use this action, add the following steps to your workflow: - Check out the current pull request using the actions/checkout action. It's recommended to set fetch-depth to 0 and submodules to recursive to ensure that all necessary dependencies are fetched. - - + Check out the current pull request using the actions/checkout action. It's recommended to set fetch-depth to 0 and submodules to 'recursive' to ensure that all necessary dependencies are fetched. ```yaml steps: @@ -19,6 +17,7 @@ This GitHub Action exports a variable that contains the list of Go modules chang with: fetch-depth: 0 submodules: 'recursive' + dependencyLevelResolution: "modules" ``` 1. Use the synapsecns/sanguine/git-changes-action Docker image to run the git-changes script, which exports a variable that contains the list of changed Go modules. @@ -29,55 +28,26 @@ This GitHub Action exports a variable that contains the list of Go modules chang with: github_token: ${{ secrets.github_token }} timeout: "1m" # optional, defaults to 1m + dependencyLevelResolution: "modules" + # For package level resolution, use "packages" instead of "modules" ``` You can customize the behavior of the git-changes script by using the following inputs: - `github_token`: The token to use for authentication with the GitHub API. This is required to fetch information about the current pull request. - `timeout`: The maximum time to wait for the GitHub API to respond. Defaults to 1 minute. + - `dependencyLevelResolution`: "modules" or "packages". The output of the git-changes script is a comma-separated list of Go module paths. You can access this list using the `filter_go` output variable, like so: ```yaml - - run: echo "Changed modules: ${{ steps.filter_go.outputs.changed_modules }}" - - run: echo "Unchanged modules: ${{ steps.filter_go.outputs.unchanged_modules }}" - - run: echo "Changed modules (including dependencies): ${{ steps.filter_go.outputs.changed_modules_deps }}" - - run: echo "Unchanged modules (including dependencies): ${{ steps.filter_go.outputs.unchanged_modules_deps }}" -``` - -## Example - -Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules: - - -```yaml -name: Test Go Modules - -on: - pull_request: - types: [opened, synchronize] - -jobs: - test_go_modules: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'recursive' + - run: echo 'Changed modules:' ${{ steps.filter_go.outputs.changed_modules }} + - run: echo 'Unchanged modules:' ${{ steps.filter_go.outputs.unchanged_modules }} + - run: echo 'Changed modules (including dependencies):' ${{ steps.filter_go.outputs.changed_modules_deps }} + - run: echo 'Unchanged modules (including dependencies):' ${{ steps.filter_go.outputs.unchanged_modules_deps }} - - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest - id: filter_go - with: - github_token: ${{ secrets.github_token }} - timeout: "1m" - - - name: Run tests - run: go test -v ${{ steps.filter_go.outputs.changed_modules }} ``` -This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. - ## How It Works First a change tree is calculated between the ref and the base (as passed in by the user or inferred by the event type) based on the flow described below and [here](https://github.com/dorny/paths-filter/blob/4067d885736b84de7c414f582ac45897079b0a78/README.md#supported-workflows). This is a file tree that contains all the files that have changed between two refs. @@ -94,7 +64,9 @@ graph TB F --> G ``` -Each module in the `go.work` is visited. If any changes were detected by the previous step, the module is added to the list of changed modules. If `include_dependencies` is on and the module has a dependency that is also in the `go.work`, the dependency is added to the list of changed modules as well. This process is repeated until all modules in the `go.work` have been visited. +## Module Level Resolution + +Each module in the `go.work` is visited. If any changes are detected, the module itself is added to the list of changed modules. By setting `include_dependencies` to `modules`, changes to their direct or indirect module dependencies will flag the module as changed. This process is repeated until all modules in the `go.work` have been visited. ```mermaid sequenceDiagram @@ -126,3 +98,42 @@ sequenceDiagram end GW->>GW: Continue Until All Modules Visited ``` + +## Package Level Resolution + +Each module in the `go.work` is visited. If any changes are detected, the module itself is added to the list of changed modules. By setting `include_dependencies` to `packages`, changes to its direct or indirect package dependencies will flag the module as changed. This process is repeated until all package dependencies for the module been visited. + +```mermaid +sequenceDiagram + participant GW as go.work + participant M as Module + participant P as Package + participant CML as Changed_Module_List + participant UML as Unchanged_Module_List + participant D as Dependency + + GW->>M: Visit Module + M->>P:Extract all packages for each Module + P->>M: Returns [module]: [package1,package2] +M->>M: Changes detected? +M->>P: Loop over packages + P-->>GW: Changes Detected? + alt Changes Detected + GW->>CML: Add Module to Changed_Module_List + M->>D: Has Package Dependency? + alt Has Dependency + GW->>CML: Add Dependency to Changed_Module_List + else No Dependency + M-->>GW: No Dependency to Add + end + else No Changes Detected + GW->>UML: Add Module to Unchanged_Module_List + M->>D: Has Dependency in go.work? + alt Has Dependency + GW->>UML: Add Dependency to Unchanged_Module_List + else No Dependency + M-->>GW: No Dependency to Add + end + end + GW->>GW: Continue Until All Modules Visited +``` diff --git a/contrib/git-changes-action/detector/doc.go b/contrib/git-changes-action/detector/doc.go deleted file mode 100644 index c87af8467c..0000000000 --- a/contrib/git-changes-action/detector/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package detector implements a detector for detecting changes in a git repo. -package detector diff --git a/contrib/git-changes-action/detector/export_test.go b/contrib/git-changes-action/detector/export_test.go deleted file mode 100644 index 40295fd6c2..0000000000 --- a/contrib/git-changes-action/detector/export_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package detector - -import ( - "github.com/go-git/go-git/v5" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/actionscore" -) - -func GetDependencyDag(repoPath string) (map[string][]string, error) { - return getDependencyGraph(repoPath) -} - -func GetHead(repo *git.Repository, ghContext *actionscore.Context, head string) (string, error) { - return getHead(repo, ghContext, head) -} diff --git a/contrib/git-changes-action/detector/git/doc.go b/contrib/git-changes-action/detector/git/doc.go new file mode 100644 index 0000000000..3ce63647eb --- /dev/null +++ b/contrib/git-changes-action/detector/git/doc.go @@ -0,0 +1,2 @@ +// Package git provides utilities to interact with git. +package git diff --git a/contrib/git-changes-action/detector/eventtype_string.go b/contrib/git-changes-action/detector/git/eventtype_string.go similarity index 98% rename from contrib/git-changes-action/detector/eventtype_string.go rename to contrib/git-changes-action/detector/git/eventtype_string.go index 7b7b71978d..3cb7df1226 100644 --- a/contrib/git-changes-action/detector/eventtype_string.go +++ b/contrib/git-changes-action/detector/git/eventtype_string.go @@ -1,6 +1,6 @@ // Code generated by "stringer -type=EventType -linecomment"; DO NOT EDIT. -package detector +package git import "strconv" diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/git/git.go similarity index 83% rename from contrib/git-changes-action/detector/detector.go rename to contrib/git-changes-action/detector/git/git.go index 97e631d228..f05a5e2656 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/git/git.go @@ -1,11 +1,10 @@ -package detector +package git import ( "encoding/hex" "encoding/json" "errors" "fmt" - "github.com/ethereum/go-ethereum/common" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" @@ -13,60 +12,10 @@ import ( "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/actionscore" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" "github.com/synapsecns/sanguine/core" - "golang.org/x/mod/modfile" "os" - "path" "strings" ) -// DetectChangedModules is the change detector client. -// nolint: cyclop -func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modules map[string]bool, err error) { - modules = make(map[string]bool) - - goWorkPath := path.Join(repoPath, "go.work") - - if !common.FileExist(goWorkPath) { - return nil, fmt.Errorf("go.work file not found in %s", repoPath) - } - - //nolint: gosec - workFile, err := os.ReadFile(goWorkPath) - if err != nil { - return nil, fmt.Errorf("failed to read go.work file: %w", err) - } - - parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) - if err != nil { - return nil, fmt.Errorf("failed to parse go.work file: %w", err) - } - - depGraph, err := getDependencyGraph(repoPath) - if err != nil { - return nil, fmt.Errorf("could not get dep graph: %w", err) - } - - for _, module := range parsedWorkFile.Use { - changed := false - if ct.HasPath(module.Path) { - changed = true - } - - if includeDeps { - deps := depGraph[module.Path] - for _, dep := range deps { - if ct.HasPath(dep) { - changed = true - } - } - } - - modules[module.Path] = changed - } - - return modules, nil -} - // getChangeTreeFromGit returns a tree of all the files that have changed between the current commit and the commit with the given hash. // nolint: cyclop, gocognit func getChangeTreeFromGit(repoPath string, ghContext *actionscore.Context, head, base string) (tree.Tree, error) { @@ -76,7 +25,7 @@ func getChangeTreeFromGit(repoPath string, ghContext *actionscore.Context, head, return nil, fmt.Errorf("could not open repository %s: %w", repoPath, err) } - head, err = getHead(repository, ghContext, head) + head, err = GetHead(repository, ghContext, head) if err != nil { return nil, fmt.Errorf("could not get head: %w", err) } @@ -272,9 +221,9 @@ func getHeadBase(repo *git.Repository) (head string, base string, err error) { return co.Hash().String(), lastCommit.Hash.String(), nil } -// getHead gets the head of the current branch. +// GetHead gets the head of the current branch. // it attempts to mirror the logic of https://github.com/dorny/paths-filter/blob/0ef5f0d812dc7b631d69e07d2491d70fcebc25c8/src/main.ts#L104 -func getHead(repo *git.Repository, ghContext *actionscore.Context, head string) (string, error) { +func GetHead(repo *git.Repository, ghContext *actionscore.Context, head string) (string, error) { if head != "" { return head, nil } diff --git a/contrib/git-changes-action/detector/ref.go b/contrib/git-changes-action/detector/git/ref.go similarity index 99% rename from contrib/git-changes-action/detector/ref.go rename to contrib/git-changes-action/detector/git/ref.go index 204db93f0a..5c8495177c 100644 --- a/contrib/git-changes-action/detector/ref.go +++ b/contrib/git-changes-action/detector/git/ref.go @@ -1,4 +1,4 @@ -package detector +package git import ( "context" diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/module/depgraph.go similarity index 70% rename from contrib/git-changes-action/detector/depgraph.go rename to contrib/git-changes-action/detector/module/depgraph.go index 39651d34ea..f4b7d2acac 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/module/depgraph.go @@ -1,4 +1,4 @@ -package detector +package moduledetector import ( "fmt" @@ -36,14 +36,15 @@ func getDependencyGraph(repoPath string) (moduleDeps map[string][]string, err er } // map of module->dependencies + replaces - var dependencies map[string][]string - // bidirectional map of module->module name - var moduleNames *bimap.BiMap[string, string] + var dependencies map[string]map[string]struct{} + // bidirectional map of module->module + // moduleRelativeName <-> modulePublicName + var dependencyNames *bimap.BiMap[string, string] // iterate through each module in the go.work file // create a list of dependencies for each module // and module names - dependencies, moduleNames, err = makeDepMaps(repoPath, parsedWorkFile.Use) + dependencies, dependencyNames, err = makeDepMaps(repoPath, parsedWorkFile.Use) if err != nil { return nil, fmt.Errorf("failed to create dependency maps: %w", err) } @@ -51,10 +52,9 @@ func getDependencyGraph(repoPath string) (moduleDeps map[string][]string, err er depGraph := depgraph.New() // build the dependency graph for _, module := range parsedWorkFile.Use { - moduleDeps := dependencies[module.Path] - for _, dep := range moduleDeps { + for dep := range dependencies[module.Path] { // check if the full package name (e.g. github.com/myorg/myrepo/mymodule) is in the list of modules. If it is, add it as a dependency after renaming - renamedDep, hasDep := moduleNames.GetInverse(dep) + renamedDep, hasDep := dependencyNames.GetInverse(dep) if hasDep { err = depGraph.DependOn(module.Path, renamedDep) if err != nil { @@ -81,41 +81,45 @@ func getDependencyGraph(repoPath string) (moduleDeps map[string][]string, err er return moduleDeps, nil } -// makeDepMaps makes a dependency map and a bidirectional map of dep<->module. -func makeDepMaps(repoPath string, uses []*modfile.Use) (dependencies map[string][]string, moduleNames *bimap.BiMap[string, string], err error) { - // map of module->dependencies + replaces - dependencies = make(map[string][]string) +// makeDepMaps builds a +// 1. module->dependency map +// 2. bidirectional map of module<->module (moduleRelativeName to modulePublicName). +func makeDepMaps(repoPath string, uses []*modfile.Use) (dependencies map[string]map[string]struct{}, dependencyNames *bimap.BiMap[string, string], err error) { + // map of module->depndencies + dependencies = make(map[string]map[string]struct{}) + // bidirectional map of module->module name - moduleNames = bimap.NewBiMap[string, string]() + // Maps relative to public names, used to filer out all external libraries/packages. + dependencyNames = bimap.NewBiMap[string, string]() // iterate through each module in the go.work file - // create a list of dependencies for each module - // and module names + // 1. Create a list of dependencies for each module + // 2. Map public names to private names for each module + //nolint: gosec for _, module := range uses { - //nolint: gosec modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) if err != nil { - return dependencies, moduleNames, fmt.Errorf("failed to read module file %s: %w", module.Path, err) + return dependencies, dependencyNames, fmt.Errorf("failed to read module file %s: %w", module.Path, err) } parsedModFile, err := modfile.Parse(module.Path, modContents, nil) if err != nil { - return dependencies, moduleNames, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) + return dependencies, dependencyNames, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - moduleNames.Insert(module.Path, parsedModFile.Module.Mod.Path) + dependencyNames.Insert(module.Path, parsedModFile.Module.Mod.Path) + dependencies[module.Path] = make(map[string]struct{}) - dependencies[module.Path] = make([]string, 0) // include all requires and replaces, as they are dependencies for _, require := range parsedModFile.Require { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.Mod.Path)) + dependencies[module.Path][convertRelPath(repoPath, module.Path, require.Mod.Path)] = struct{}{} } for _, require := range parsedModFile.Replace { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.New.Path)) + dependencies[module.Path][convertRelPath(repoPath, module.Path, require.New.Path)] = struct{}{} } } - return dependencies, moduleNames, nil + return dependencies, dependencyNames, nil } // isRelativeDep returns true if the dependency is relative to the module (starts with ./ or ../). diff --git a/contrib/git-changes-action/detector/module/detector.go b/contrib/git-changes-action/detector/module/detector.go new file mode 100644 index 0000000000..1873744339 --- /dev/null +++ b/contrib/git-changes-action/detector/module/detector.go @@ -0,0 +1,59 @@ +package moduledetector + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" + "golang.org/x/mod/modfile" + "os" + "path" +) + +// DetectChangedModules is the change detector client. +// Will flag modules as changed if any module in their dependency tree has been modified. +// nolint: cyclop +func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modules map[string]bool, err error) { + modules = make(map[string]bool) + + goWorkPath := path.Join(repoPath, "go.work") + + if !common.FileExist(goWorkPath) { + return nil, fmt.Errorf("go.work file not found in %s", repoPath) + } + + //nolint: gosec + workFile, err := os.ReadFile(goWorkPath) + if err != nil { + return nil, fmt.Errorf("failed to read go.work file: %w", err) + } + + parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse go.work file: %w", err) + } + + depGraph, err := getDependencyGraph(repoPath) + if err != nil { + return nil, fmt.Errorf("could not get dep graph: %w", err) + } + + for _, module := range parsedWorkFile.Use { + changed := false + if ct.HasPath(module.Path) { + changed = true + } + + if includeDeps { + deps := depGraph[module.Path] + for _, dep := range deps { + if ct.HasPath(dep) { + changed = true + } + } + } + + modules[module.Path] = changed + } + + return modules, nil +} diff --git a/contrib/git-changes-action/detector/detector_test.go b/contrib/git-changes-action/detector/module/detector_test.go similarity index 76% rename from contrib/git-changes-action/detector/detector_test.go rename to contrib/git-changes-action/detector/module/detector_test.go index 5093168102..699f750ec0 100644 --- a/contrib/git-changes-action/detector/detector_test.go +++ b/contrib/git-changes-action/detector/module/detector_test.go @@ -1,18 +1,21 @@ -package detector_test +package moduledetector_test import ( "encoding/hex" + "fmt" "github.com/ethereum/go-ethereum/crypto" . "github.com/stretchr/testify/assert" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/actionscore" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/git" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/gitmock" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/module" "os" "path/filepath" ) func (d *DetectorSuite) TestChangedModules() { testRepo, err := gitmock.NewTestRepo(d.T(), d.sourceRepo.repo, d.sourceRepo.dir) + fmt.Println("TEST REPO: ", testRepo) Nil(d.T(), err, "should not return an error") _, err = os.Create(filepath.Join(d.sourceRepo.dir, "lib", "newfile.go")) @@ -20,13 +23,14 @@ func (d *DetectorSuite) TestChangedModules() { testRepo.Commit() - ct, err := detector.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") + ct, err := git.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") + fmt.Println(ct) Nil(d.T(), err, "should not return an error") - withDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, true) + withDeps, err := moduledetector.DetectChangedModules(d.sourceRepo.dir, ct, true) Nil(d.T(), err, "should not return an error") - withoutDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, false) + withoutDeps, err := moduledetector.DetectChangedModules(d.sourceRepo.dir, ct, false) Nil(d.T(), err, "should not return an error") False(d.T(), withoutDeps["./cmd/app1"]) @@ -41,7 +45,7 @@ func (d *DetectorSuite) TestChangedModules() { } func (d *DetectorSuite) TestGetDependencyDag() { - deps, err := detector.GetDependencyDag(d.sourceRepo.dir) + deps, err := moduledetector.GetDependencyDag(d.sourceRepo.dir) Nil(d.T(), err, "should not return an error") Equal(d.T(), deps["./cmd/app1"], []string{"./lib"}) @@ -57,14 +61,14 @@ func (d *DetectorSuite) TestGetHead() { hash := testRepo.Commit() - head, err := detector.GetHead(d.sourceRepo.repo, &actionscore.Context{ + head, err := git.GetHead(d.sourceRepo.repo, &actionscore.Context{ Ref: hash, }, "") Nil(d.T(), err, "should not return an error") Equal(d.T(), head, hash) randHash := hex.EncodeToString(crypto.Keccak256([]byte("random hash"))) - head, err = detector.GetHead(d.sourceRepo.repo, &actionscore.Context{ + head, err = git.GetHead(d.sourceRepo.repo, &actionscore.Context{ Ref: hash, }, randHash) Nil(d.T(), err, "should not return an error") @@ -77,7 +81,7 @@ func (d *DetectorSuite) TestChangeTree() { addedFiles := testRepo.AddRandomFiles(5) - changeTree, err := detector.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") + changeTree, err := git.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") Nil(d.T(), err, "should not empty change tree") for _, file := range addedFiles { diff --git a/contrib/git-changes-action/detector/module/doc.go b/contrib/git-changes-action/detector/module/doc.go new file mode 100644 index 0000000000..cbf76017ad --- /dev/null +++ b/contrib/git-changes-action/detector/module/doc.go @@ -0,0 +1,3 @@ +// Package moduledetector implements a detector for detecting changes in a git repo. +// Dependencies are transitive, at the module level. Modules depend on modules. +package moduledetector diff --git a/contrib/git-changes-action/detector/module/export_test.go b/contrib/git-changes-action/detector/module/export_test.go new file mode 100644 index 0000000000..bf1f265434 --- /dev/null +++ b/contrib/git-changes-action/detector/module/export_test.go @@ -0,0 +1,5 @@ +package moduledetector + +func GetDependencyDag(repoPath string) (map[string][]string, error) { + return getDependencyGraph(repoPath) +} diff --git a/contrib/git-changes-action/detector/suite_test.go b/contrib/git-changes-action/detector/module/suite_test.go similarity index 99% rename from contrib/git-changes-action/detector/suite_test.go rename to contrib/git-changes-action/detector/module/suite_test.go index 494a267978..c8235ca99e 100644 --- a/contrib/git-changes-action/detector/suite_test.go +++ b/contrib/git-changes-action/detector/module/suite_test.go @@ -1,4 +1,4 @@ -package detector_test +package moduledetector_test import ( "github.com/Flaque/filet" diff --git a/contrib/git-changes-action/detector/package/depgraph.go b/contrib/git-changes-action/detector/package/depgraph.go new file mode 100644 index 0000000000..a6898e3d0a --- /dev/null +++ b/contrib/git-changes-action/detector/package/depgraph.go @@ -0,0 +1,205 @@ +package packagedetector + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/kendru/darwin/go/depgraph" + "github.com/vishalkuo/bimap" + "go/parser" + "go/token" + "golang.org/x/mod/modfile" + "os" + "path" + "path/filepath" + "strings" +) + +// nolint: cyclop +func getPackageDependencyGraph(repoPath string) (moduleDeps map[string][]string, packagesPerModule map[string][]string, err error) { + moduleDeps = make(map[string][]string) + // parse the go.work file + goWorkPath := path.Join(repoPath, "go.work") + + if !common.FileExist(goWorkPath) { + return nil, nil, fmt.Errorf("go.work file not found in %s", repoPath) + } + + //nolint: gosec + workFile, err := os.ReadFile(goWorkPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read go.work file: %w", err) + } + + parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse go.work file: %w", err) + } + + // map of package->dependencies + var dependencies map[string]map[string]struct{} + + // iterate through each module in the go.work file + // create a list of dependencies for each package + // and generate a list of packages per module + // nolint: gocognit + dependencies, packagesPerModule, err = makePackageDepMaps(repoPath, parsedWorkFile.Use) + if err != nil { + return nil, nil, fmt.Errorf("failed to create dependency maps: %w", err) + } + + depGraph := depgraph.New() + + for _, module := range parsedWorkFile.Use { + for _, relativePackageName := range packagesPerModule[module.Path] { + for relativePackageDependencyName := range dependencies[relativePackageName] { + err = depGraph.DependOn(relativePackageName, relativePackageDependencyName) + // Circular dependencies are fine as long as both packages are in the same module + if err != nil && !(strings.Contains(relativePackageDependencyName, module.Path) && strings.Contains(relativePackageName, module.Path)) { + return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", relativePackageName, relativePackageDependencyName, err) + } + } + } + } + + for _, module := range parsedWorkFile.Use { + for _, relativePackageName := range packagesPerModule[module.Path] { + for dep := range depGraph.Dependencies(relativePackageName) { + moduleDeps[relativePackageName] = append(moduleDeps[relativePackageName], dep) + } + } + } + return moduleDeps, packagesPerModule, nil +} + +func extractGoFileNames(pwd string, currentPackage string, goFiles map[string][]string) (err error) { + searchNext := make(map[string]string) + _, packageDir := path.Split(currentPackage) + searchNext[pwd] = packageDir + + for len(searchNext) > 0 { + discovered := make(map[string]string) + for path, dirName := range searchNext { + // see https://stackoverflow.com/a/68559720 + dirName := dirName + path := path + err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() && !(path == filePath) { + discovered[filePath] = info.Name() + return filepath.SkipDir + } else if strings.HasSuffix(info.Name(), ".go") { + goFiles["/"+dirName] = append(goFiles["/"+dirName], filePath) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to walk path: %w", err) + } + } + searchNext = discovered + } + return nil +} + +// nolint: gocognit, cyclop +func makePackageDepMaps(repoPath string, uses []*modfile.Use) (dependencies map[string]map[string]struct{}, packagesPerModule map[string][]string, err error) { + // map of packages -> dependencies + dependencies = make(map[string]map[string]struct{}) + + // bidirectional map of package->package name + // Maps relative to public names, used to filer out all external libraries/packages. + dependencyNames := bimap.NewBiMap[string, string]() + + // map of module->packages + packagesPerModule = make(map[string][]string) + + // map module->package->goFiles + // Maps each module to all packages and each package to its go files. + extractedGoFileNames := make(map[string]map[string][]string) + + pwd, err := os.Getwd() + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to read current directory: %w", err) + } + // iterate through each module in the go.work file + // 1. Extract all go files filepaths for each package. + // 2. Create a map where key is module, value is an array with all packages in the module + // 3. Map public name to relative name for each package (used to filter external library/package imports) + for _, module := range uses { + // nolint: gosec + modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) + } + + parsedModFile, err := modfile.Parse(module.Path, modContents, nil) + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) + } + + extractedGoFileNames[module.Path] = make(map[string][]string) + err = extractGoFileNames(pwd+module.Path[1:], module.Path[1:], extractedGoFileNames[module.Path]) + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to extract go files for module %s: %w", module.Path, err) + } + + for packageName := range extractedGoFileNames[module.Path] { + var relativePackageName string + if strings.HasSuffix(module.Path, packageName) { + relativePackageName = module.Path + } else { + relativePackageName = module.Path + packageName + } + + var publicPackageName string + if strings.HasSuffix(parsedModFile.Module.Mod.Path, packageName) { + publicPackageName = parsedModFile.Module.Mod.Path + } else { + publicPackageName = parsedModFile.Module.Mod.Path + packageName + } + + packagesPerModule[module.Path] = append(packagesPerModule[module.Path], relativePackageName) + dependencyNames.Insert(relativePackageName, publicPackageName) + } + } + + // iterate through each module in the go.work file + // For every package in the module + // using the filepaths extracted on the previous loop, parse files and extract imports. + // Ignore any external library/package imports + for _, module := range uses { + for packageInModule, files := range extractedGoFileNames[module.Path] { + var relativePackageName string + if strings.HasSuffix(module.Path, packageInModule) { + relativePackageName = module.Path + } else { + relativePackageName = module.Path + packageInModule + } + + dependencies[relativePackageName] = make(map[string]struct{}) + for _, file := range files { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to parse go file %s in package %s: %w", file, relativePackageName, err) + } + + for _, s := range f.Imports { + // s.Path.Value contains double quotation marks that must be removed before indexing dependencyNames + renamedDep, hasDep := dependencyNames.GetInverse(s.Path.Value[1 : len(s.Path.Value)-1]) + + if hasDep && (relativePackageName != renamedDep) { + dependencies[relativePackageName][renamedDep] = struct{}{} + } + } + } + } + } + return dependencies, packagesPerModule, nil +} diff --git a/contrib/git-changes-action/detector/package/detector.go b/contrib/git-changes-action/detector/package/detector.go new file mode 100644 index 0000000000..0f8fd52cb2 --- /dev/null +++ b/contrib/git-changes-action/detector/package/detector.go @@ -0,0 +1,76 @@ +package packagedetector + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" + "golang.org/x/mod/modfile" + "os" + "path" +) + +// DetectChangedModules is the change detector client. +// Modules will be flagged as changed if any of the packaged in their dependency tree has been modified. +// nolint: cyclop +func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modules map[string]bool, err error) { + modules = make(map[string]bool) + + goWorkPath := path.Join(repoPath, "go.work") + + if !common.FileExist(goWorkPath) { + return nil, fmt.Errorf("go.work file not found in %s", repoPath) + } + + //nolint: gosec + workFile, err := os.ReadFile(goWorkPath) + if err != nil { + return nil, fmt.Errorf("failed to read go.work file: %w", err) + } + + parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse go.work file: %w", err) + } + + depGraph, packagesPerModule, err := getPackageDependencyGraph(repoPath) + if err != nil { + return nil, fmt.Errorf("could not get dep graph: %w", err) + } + + for _, module := range parsedWorkFile.Use { + changed := false + + if ct.HasPath(module.Path) { + changed = true + } + + if includeDeps && !changed { + for _, packageName := range packagesPerModule[module.Path] { + if isPackageChanged(packageName, ct, depGraph) { + changed = true + // If a package is flagged as changed + // its not necessary to analyze the remaining packages, + // module can be flagged as changed + break + } + } + } + + modules[module.Path] = changed + } + + return modules, nil +} + +func isPackageChanged(packageName string, ct tree.Tree, depGraph map[string][]string) bool { + if ct.HasPath(packageName) { + return true + } + + for _, dep := range depGraph[packageName] { + if ct.HasPath(dep) { + return true + } + } + return false +} diff --git a/contrib/git-changes-action/detector/package/doc.go b/contrib/git-changes-action/detector/package/doc.go new file mode 100644 index 0000000000..085a40b791 --- /dev/null +++ b/contrib/git-changes-action/detector/package/doc.go @@ -0,0 +1,3 @@ +// Package packagedetector implements a detector for detecting changes in a git repo. +// Dependencies are transitive, at the package level. Modules depend on packages, whose dep graph is also at package level. Module->package->package +package packagedetector diff --git a/contrib/git-changes-action/main.go b/contrib/git-changes-action/main.go index 60e2ea6883..a4a21ec34a 100644 --- a/contrib/git-changes-action/main.go +++ b/contrib/git-changes-action/main.go @@ -5,20 +5,24 @@ import ( "context" "encoding/json" "fmt" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" "os" "sort" "strings" "time" + moduledetector "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/module" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/package" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" + "github.com/sethvargo/go-githubactions" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/git" ) const defaultTimeout = "1m" func main() { token := githubactions.GetInput("github_token") + dependencyLevelResolution := githubactions.GetInput("dependencyLevelResolution") workingDirectory, err := os.Getwd() if err != nil { @@ -36,17 +40,17 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - ct, err := detector.GetChangeTree(ctx, workingDirectory, ref, token, base) + ct, err := git.GetChangeTree(ctx, workingDirectory, ref, token, base) if err != nil { panic(err) } - noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false) + noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false, dependencyLevelResolution) if err != nil { panic(err) } - depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true) + depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true, dependencyLevelResolution) if err != nil { panic(err) } @@ -61,10 +65,17 @@ func main() { // outputModuleChanges outputs the changed modules. // this wraps detector.DetectChangedModules and handles the output formatting to be parsable by github actions. // the final output is a json array of strings. -func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool) (changedJSON string, unchangedJson string, err error) { - modules, err := detector.DetectChangedModules(workingDirectory, ct, includeDeps) +func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool, dependencyLevelResolution string) (changedJSON string, unchangedJSON string, err error) { + var modules map[string]bool + + if dependencyLevelResolution == "packages" { + modules, err = packagedetector.DetectChangedModules(workingDirectory, ct, includeDeps) + } else { + modules, err = moduledetector.DetectChangedModules(workingDirectory, ct, includeDeps) + } + if err != nil { - return changedJSON, unchangedJson, fmt.Errorf("failed to detect changed modules w/ include deps set to %v: %w", includeDeps, err) + return changedJSON, unchangedJSON, fmt.Errorf("failed to detect changed modules w/ include deps set to %v: %w", includeDeps, err) } var changedModules, unchangedModules []string @@ -83,12 +94,12 @@ func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool marshalledChanged, err := json.Marshal(changedModules) if err != nil { - return changedJSON, unchangedJson, fmt.Errorf("failed to marshall changed module json w/ include deps set to %v: %w", includeDeps, err) + return changedJSON, unchangedJSON, fmt.Errorf("failed to marshall changed module json w/ include deps set to %v: %w", includeDeps, err) } marshalledUnchanged, err := json.Marshal(unchangedModules) if err != nil { - return changedJSON, unchangedJson, fmt.Errorf("failed to marshall unchanged module json w/ include deps set to %v: %w", includeDeps, err) + return changedJSON, unchangedJSON, fmt.Errorf("failed to marshall unchanged module json w/ include deps set to %v: %w", includeDeps, err) } return string(marshalledChanged), string(marshalledUnchanged), nil