From c11c90d3c5e3366980de4eac78080d85b18c8e83 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sat, 23 Mar 2024 01:21:59 -0600 Subject: [PATCH 01/39] feat: add package-based dependency resolution --- .../git-changes-action/detector/depgraph.go | 216 +++++++++++++----- .../git-changes-action/detector/detector.go | 91 ++++++-- contrib/git-changes-action/main.go | 8 +- 3 files changed, 236 insertions(+), 79 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 39651d34ea..e7a470a199 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -2,120 +2,220 @@ package detector import ( "fmt" - "github.com/ethereum/go-ethereum/common" - "github.com/kendru/darwin/go/depgraph" - "github.com/vishalkuo/bimap" - "golang.org/x/mod/modfile" + "go/parser" + "go/token" "os" "path" "path/filepath" "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/kendru/darwin/go/depgraph" + "github.com/vishalkuo/bimap" + "golang.org/x/mod/modfile" ) // getDependencyGraph returns a dependency graph of all the modules in the go.work file that refer to other modules in the go.work file // returns a map of module (./my_module)->(./my_module_dependency1,./my_module_dependency2). // nolint: cyclop -func getDependencyGraph(repoPath string) (moduleDeps map[string][]string, err error) { +func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps map[string][]string, packagesPerModule map[string][]string, err error) { moduleDeps = make(map[string][]string) // parse the go.work file goWorkPath := path.Join(repoPath, "go.work") if !common.FileExist(goWorkPath) { - return nil, fmt.Errorf("go.work file not found in %s", repoPath) + return nil, nil, fmt.Errorf("go.work file not found in %s", repoPath) } //nolint: gosec workFile, err := os.ReadFile(goWorkPath) if err != nil { - return nil, fmt.Errorf("failed to read go.work file: %w", err) + return nil, nil, fmt.Errorf("failed to read go.work file: %w", err) } parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) if err != nil { - return nil, fmt.Errorf("failed to parse go.work file: %w", err) + return nil, nil, fmt.Errorf("failed to parse go.work file: %w", err) } // map of module->dependencies + replaces var dependencies map[string][]string - // bidirectional map of module->module name - var moduleNames *bimap.BiMap[string, string] + // bidirectional map of module->module name or dependency->dependency + var dependencyNames *bimap.BiMap[string, string] - // iterate through each module in the go.work file + // iterate through each module in the go.work file // create a list of dependencies for each module // and module names - dependencies, moduleNames, err = makeDepMaps(repoPath, parsedWorkFile.Use) + dependencies, dependencyNames, packagesPerModule, err = makeDepMaps(repoPath, parsedWorkFile.Use, typeOfDependency) + if err != nil { - return nil, fmt.Errorf("failed to create dependency maps: %w", err) + return nil, nil, fmt.Errorf("failed to create dependency maps: %w", err) } depGraph := depgraph.New() // build the dependency graph - for _, module := range parsedWorkFile.Use { - moduleDeps := dependencies[module.Path] - for _, dep := range moduleDeps { - // check if the full package name (e.g. github.com/myorg/myrepo/mymodule) is in the list of modules. If it is, add it as a dependency after renaming - renamedDep, hasDep := moduleNames.GetInverse(dep) - if hasDep { - err = depGraph.DependOn(module.Path, renamedDep) - if err != nil { - return nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) - } - } - - if isRelativeDep(dep) { - // if the dependency is relative, add it as a dependency - err = depGraph.DependOn(module.Path, dep) - if err != nil { - return nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) - } - } - } - } - - for _, module := range parsedWorkFile.Use { - for dep := range depGraph.Dependencies(module.Path) { - moduleDeps[module.Path] = append(moduleDeps[module.Path], dep) - } - } + if typeOfDependency == "module" { + for _, module := range parsedWorkFile.Use { + moduleDeps := dependencies[module.Path] + for _, dep := range moduleDeps { + // check if the full package name (e.g. github.com/myorg/myrepo/mymodule) is in the list of modules. If it is, add it as a dependency after renaming + renamedDep, hasDep := dependencyNames.GetInverse(dep) + if hasDep { + err = depGraph.DependOn(module.Path, renamedDep) + if err != nil { + return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) + } + } + + if isRelativeDep(dep) { + // if the dependency is relative, add it as a dependency + err = depGraph.DependOn(module.Path, dep) + if err != nil { + return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) + } + } + } + } + + for _, module := range parsedWorkFile.Use { + for dep := range depGraph.Dependencies(module.Path) { + moduleDeps[module.Path] = append(moduleDeps[module.Path], dep) + } + } + } + + if typeOfDependency == "packages" { + for _, module := range parsedWorkFile.Use { + allPackagesInModule := packagesPerModule[module.Path] + + for _, packageInModule := range allPackagesInModule { + renamedPackage, hasPackage := dependencyNames.Get(packageInModule) + t := dependencies[renamedPackage] + + if hasPackage { + for _, dep := range t { + dep = strings.TrimPrefix(dep, `"`) + dep = strings.TrimSuffix(dep, `"`) + + renamedDep, hasDep := dependencyNames.GetInverse(dep) + if hasDep { + err = depGraph.DependOn(packageInModule, renamedDep) + if err != nil { + fmt.Println("THERE IS AN ERROR", err, packageInModule, renamedDep) + } + } + } + } + + for dep := range depGraph.Dependencies(packageInModule) { + moduleDeps[packageInModule] = append(moduleDeps[packageInModule], dep) + } + } + } + } + + return moduleDeps, packagesPerModule, nil +} - return moduleDeps, nil +func extractGoFileNames(pwd string, currentModule string, currentPackage string, goFiles map[string][]string) { + ls, err := os.ReadDir(pwd) + if err != nil { + } + + for _, entry := range ls { + if entry.IsDir() { + extractGoFileNames(pwd + "/" + entry.Name(), currentModule, entry.Name(), goFiles) + } else if strings.Contains(entry.Name(), ".go") { + fileName := pwd + "/" + entry.Name() + goFiles["/" + currentPackage] = append(goFiles["/" + currentModule + "/" + currentPackage], fileName) + } + } } // makeDepMaps makes a dependency map and a bidirectional map of dep<->module. -func makeDepMaps(repoPath string, uses []*modfile.Use) (dependencies map[string][]string, moduleNames *bimap.BiMap[string, string], err error) { +func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) (dependencies map[string][]string, dependencyNames *bimap.BiMap[string, string], packagesPerModule map[string][]string, err error) { // map of module->dependencies + replaces + // map of packages -> dependencies dependencies = make(map[string][]string) // bidirectional map of module->module name - moduleNames = bimap.NewBiMap[string, string]() + // bidirectional map of package->package name, relative to public names. + dependencyNames = bimap.NewBiMap[string, string]() + // map of module->packages + packagesPerModule = make(map[string][]string) // iterate through each module in the go.work file - // create a list of dependencies for each module - // and module names + // create a list of dependencies for each module based on modules or packages + // and module names or package names for _, module := range uses { //nolint: gosec modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) if err != nil { - return dependencies, moduleNames, fmt.Errorf("failed to read module file %s: %w", module.Path, err) + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) } parsedModFile, err := modfile.Parse(module.Path, modContents, nil) if err != nil { - return dependencies, moduleNames, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - moduleNames.Insert(module.Path, parsedModFile.Module.Mod.Path) - - dependencies[module.Path] = make([]string, 0) - // include all requires and replaces, as they are dependencies - for _, require := range parsedModFile.Require { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.Mod.Path)) - } - for _, require := range parsedModFile.Replace { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.New.Path)) - } + if typeOfDependency == "module" { + dependencyNames.Insert(module.Path, parsedModFile.Module.Mod.Path) + + dependencies[module.Path] = make([]string, 0) + // include all requires and replaces, as they are dependencies + for _, require := range parsedModFile.Require { + dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.Mod.Path)) + } + for _, require := range parsedModFile.Replace { + dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.New.Path)) + } + } + + + + if typeOfDependency == "packages" { + extractedGoFileNames := make(map[string][]string) + + pwd, err := os.Getwd() + if err != nil { + } + + extractGoFileNames(pwd + module.Path[1:], module.Path[2:], module.Path[2:], extractedGoFileNames) + + fset := token.NewFileSet() + + for packageName, packageFiles := range extractedGoFileNames { + var relativePackageName string + if module.Path[1:] == packageName { + relativePackageName = packageName + } else { + relativePackageName = module.Path[1:] + packageName + } + + var publicPackageName string + if strings.Contains(parsedModFile.Module.Mod.Path, packageName) { + publicPackageName = parsedModFile.Module.Mod.Path + } else { + publicPackageName = parsedModFile.Module.Mod.Path + packageName + } + + packagesPerModule[module.Path] = append(packagesPerModule[module.Path], relativePackageName) + dependencyNames.Insert(relativePackageName, publicPackageName) + + for _, file := range packageFiles { + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + if err != nil { + } + + for _, s := range f.Imports { + dependencies[publicPackageName] = append(dependencies[publicPackageName], s.Path.Value) + } + } + } + } } - return dependencies, moduleNames, nil + return dependencies, dependencyNames, packagesPerModule, nil } // isRelativeDep returns true if the dependency is relative to the module (starts with ./ or ../). diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index 97e631d228..fabb52011e 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -19,9 +19,38 @@ import ( "strings" ) + +func identifyNestedDependencyChange(packageName string, depGraph map[string][]string, ct tree.Tree) (changed bool) { + if ct.HasPath(packageName) { + return true + } + deps := depGraph[packageName] + changed = false + + for _, dep := range deps { + if ct.HasPath(dep) { + changed = true + break + } + + subDeps := depGraph[dep] + + if len(subDeps) != 0 { + for _, subDep := range subDeps { + changed = identifyNestedDependencyChange(subDep, depGraph, ct) + + if changed == true { + break + } + } + } + } + + return changed +} // DetectChangedModules is the change detector client. // nolint: cyclop -func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modules map[string]bool, err error) { +func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeOfDependency string) (modules map[string]bool, err error) { modules = make(map[string]bool) goWorkPath := path.Join(repoPath, "go.work") @@ -41,28 +70,56 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modu return nil, fmt.Errorf("failed to parse go.work file: %w", err) } - depGraph, err := getDependencyGraph(repoPath) + depGraph, packagesPerModule, err := getDependencyGraph(repoPath, typeOfDependency) if err != nil { return nil, fmt.Errorf("could not get dep graph: %w", err) } - for _, module := range parsedWorkFile.Use { - changed := false - if ct.HasPath(module.Path) { - changed = true - } + if (typeOfDependency == "modules") { + for _, module := range parsedWorkFile.Use { + changed := false + if ct.HasPath(module.Path) { + changed = true + } - if includeDeps { - deps := depGraph[module.Path] - for _, dep := range deps { - if ct.HasPath(dep) { - changed = true - } - } - } + if includeDeps { + deps := depGraph[module.Path] + for _, dep := range deps { + if ct.HasPath(dep) { + changed = true + } + } + } - modules[module.Path] = changed - } + modules[module.Path] = changed + } + } + + if typeOfDependency == "packages" { + for _, module := range parsedWorkFile.Use { + changed := false + + if ct.HasPath(module.Path) { + changed = true + modules[module.Path] = changed + + continue + } + + if includeDeps { + for _, packageName := range packagesPerModule[module.Path] { + changed = identifyNestedDependencyChange(packageName, depGraph, ct) + + if changed == true { + break + } + } + } + + + modules[module.Path] = changed + } + } return modules, nil } diff --git a/contrib/git-changes-action/main.go b/contrib/git-changes-action/main.go index 60e2ea6883..5d02e1ab58 100644 --- a/contrib/git-changes-action/main.go +++ b/contrib/git-changes-action/main.go @@ -41,12 +41,12 @@ func main() { panic(err) } - noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false) + noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false, "packages") if err != nil { panic(err) } - depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true) + depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true, "packages") if err != nil { panic(err) } @@ -61,8 +61,8 @@ func main() { // outputModuleChanges outputs the changed modules. // this wraps detector.DetectChangedModules and handles the output formatting to be parsable by github actions. // the final output is a json array of strings. -func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool) (changedJSON string, unchangedJson string, err error) { - modules, err := detector.DetectChangedModules(workingDirectory, ct, includeDeps) +func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool, typeOfDependency string) (changedJSON string, unchangedJson string, err error) { + modules, err := detector.DetectChangedModules(workingDirectory, ct, includeDeps, typeOfDependency) if err != nil { return changedJSON, unchangedJson, fmt.Errorf("failed to detect changed modules w/ include deps set to %v: %w", includeDeps, err) } From b86fd2be6d689cd2d3b87d8e4885a88512fb1fcc Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sat, 23 Mar 2024 20:20:51 -0600 Subject: [PATCH 02/39] fix: remove redundant step in recursion --- .../git-changes-action/detector/detector.go | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index fabb52011e..f8148d3537 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -24,24 +24,16 @@ func identifyNestedDependencyChange(packageName string, depGraph map[string][]st if ct.HasPath(packageName) { return true } + deps := depGraph[packageName] changed = false - for _, dep := range deps { - if ct.HasPath(dep) { - changed = true - break - } - - subDeps := depGraph[dep] + if len(deps) != 0 { + for _, dep := range deps { + changed = identifyNestedDependencyChange(dep, depGraph, ct) - if len(subDeps) != 0 { - for _, subDep := range subDeps { - changed = identifyNestedDependencyChange(subDep, depGraph, ct) - - if changed == true { - break - } + if changed == true { + break } } } From 3b0b02c5d531af12cb19d975c668e70359235d04 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sat, 23 Mar 2024 21:12:27 -0600 Subject: [PATCH 03/39] feat: add recursion cache avoids redundant recursion on previously analyzed packages --- .../git-changes-action/detector/detector.go | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index f8148d3537..8cbdd3161c 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -20,17 +20,22 @@ import ( ) -func identifyNestedDependencyChange(packageName string, depGraph map[string][]string, ct tree.Tree) (changed bool) { +func identifyNestedDependencyChange(packageName string, depGraph map[string][]string, ct tree.Tree, packages map[string]bool) (changed bool) { if ct.HasPath(packageName) { + packages[packageName] = true return true } + if _, ok := packages[packageName]; ok { + return packages[packageName] + } + deps := depGraph[packageName] changed = false if len(deps) != 0 { for _, dep := range deps { - changed = identifyNestedDependencyChange(dep, depGraph, ct) + changed = identifyNestedDependencyChange(dep, depGraph, ct, packages) if changed == true { break @@ -38,6 +43,7 @@ func identifyNestedDependencyChange(packageName string, depGraph map[string][]st } } + packages[packageName] = changed return changed } // DetectChangedModules is the change detector client. @@ -88,19 +94,25 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO } if typeOfDependency == "packages" { + // This map stores package change identification + // map(package->isChanged bool) + // Reduces reduntant recursion over packages whose dependency tree has been prevoziously analyzed. + // Essentially caching recursion, so if package was found and analyzed through another package's dependency tree, it is not analyzed again. + var packages = make(map[string]bool) + for _, module := range parsedWorkFile.Use { changed := false if ct.HasPath(module.Path) { changed = true modules[module.Path] = changed - + packages[module.Path] = changed continue } if includeDeps { for _, packageName := range packagesPerModule[module.Path] { - changed = identifyNestedDependencyChange(packageName, depGraph, ct) + changed = identifyNestedDependencyChange(packageName, depGraph, ct, packages) if changed == true { break @@ -108,9 +120,10 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO } } - + packages[module.Path] = changed modules[module.Path] = changed } + } return modules, nil From 53300cd171afe4b565b314fc99b191dbb86c4af1 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sun, 24 Mar 2024 19:17:41 -0600 Subject: [PATCH 04/39] refactor: separate dependency extraction into 2 steps Step 1: 1. Get all packages per module 2. Get all package names publicName<->relativeName Step 2: 1. Extracts dependencies This is in preparation for the next commit which will make dependencies per package a set (to avoid dupllicates), and will add dependency filtering to step 2, so that only synapsecns/sanguine packages are included --- .../git-changes-action/detector/depgraph.go | 105 +++++++++++------- 1 file changed, 63 insertions(+), 42 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index e7a470a199..01083ef7ac 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -127,7 +127,13 @@ func extractGoFileNames(pwd string, currentModule string, currentPackage string, extractGoFileNames(pwd + "/" + entry.Name(), currentModule, entry.Name(), goFiles) } else if strings.Contains(entry.Name(), ".go") { fileName := pwd + "/" + entry.Name() - goFiles["/" + currentPackage] = append(goFiles["/" + currentModule + "/" + currentPackage], fileName) + var packageName string + if currentModule == currentPackage { + packageName = "/" + currentPackage + } else { + packageName = "/" + currentModule + "/" + currentPackage + } + goFiles[packageName] = append(goFiles[packageName], fileName) } } } @@ -146,74 +152,89 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) // iterate through each module in the go.work file // create a list of dependencies for each module based on modules or packages // and module names or package names - for _, module := range uses { - //nolint: gosec - modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) - } - - parsedModFile, err := modfile.Parse(module.Path, modContents, nil) - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) - } + //nolint: gose if typeOfDependency == "module" { - dependencyNames.Insert(module.Path, parsedModFile.Module.Mod.Path) + for _, module := range uses { + modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) + if err != nil { + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) + } - dependencies[module.Path] = make([]string, 0) - // include all requires and replaces, as they are dependencies - for _, require := range parsedModFile.Require { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.Mod.Path)) - } - for _, require := range parsedModFile.Replace { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.New.Path)) + parsedModFile, err := modfile.Parse(module.Path, modContents, nil) + if err != nil { + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) + } + + dependencyNames.Insert(module.Path, parsedModFile.Module.Mod.Path) + dependencies[module.Path] = make([]string, 0) + + // include all requires and replaces, as they are dependencies + for _, require := range parsedModFile.Require { + dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.Mod.Path)) + } + for _, require := range parsedModFile.Replace { + dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.New.Path)) + } } } if typeOfDependency == "packages" { - extractedGoFileNames := make(map[string][]string) + extractedGoFileNames := make(map[string]map[string][]string) pwd, err := os.Getwd() if err != nil { } - extractGoFileNames(pwd + module.Path[1:], module.Path[2:], module.Path[2:], extractedGoFileNames) - fset := token.NewFileSet() + for _, module := range uses { + extractedGoFileNames[module.Path[1:]] = make(map[string][]string) - for packageName, packageFiles := range extractedGoFileNames { - var relativePackageName string - if module.Path[1:] == packageName { - relativePackageName = packageName - } else { - relativePackageName = module.Path[1:] + packageName + modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) + if err != nil { + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) } - var publicPackageName string - if strings.Contains(parsedModFile.Module.Mod.Path, packageName) { - publicPackageName = parsedModFile.Module.Mod.Path - } else { - publicPackageName = parsedModFile.Module.Mod.Path + packageName + parsedModFile, err := modfile.Parse(module.Path, modContents, nil) + if err != nil { + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - packagesPerModule[module.Path] = append(packagesPerModule[module.Path], relativePackageName) - dependencyNames.Insert(relativePackageName, publicPackageName) + extractGoFileNames(pwd + module.Path[1:], module.Path[2:], module.Path[2:], extractedGoFileNames[module.Path[1:]]) - for _, file := range packageFiles { - f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) - if err != nil { + for relativePackageName, _ := range extractedGoFileNames[module.Path[1:]] { + var publicPackageName string + if strings.Contains(parsedModFile.Module.Mod.Path, relativePackageName) { + publicPackageName = parsedModFile.Module.Mod.Path + } else { + publicPackageName = parsedModFile.Module.Mod.Path + relativePackageName } - for _, s := range f.Imports { - dependencies[publicPackageName] = append(dependencies[publicPackageName], s.Path.Value) + packagesPerModule[module.Path] = append(packagesPerModule[module.Path], relativePackageName) + dependencyNames.Insert(relativePackageName, publicPackageName) + } + } + + for _, module := range uses { + for _, packageInModule := range packagesPerModule[module.Path] { + for _, file := range extractedGoFileNames[module.Path[1:]][packageInModule] { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + if err != nil { + } + + publicPackageName, _ := dependencyNames.Get(packageInModule) + for _, s := range f.Imports { + dependencies[publicPackageName] = append(dependencies[publicPackageName], s.Path.Value) + } } } } } - } + + fmt.Println(dependencies) return dependencies, dependencyNames, packagesPerModule, nil } From 1c8201f494a7c3e0be8a4426a9220f7ae6dd0ef4 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sun, 24 Mar 2024 19:32:25 -0600 Subject: [PATCH 05/39] refactor: change dependencies type Changed from map[string][]string to map[string]map[string]struct{}. Will essentially work as a set to avoid any dependency duplication --- .../git-changes-action/detector/depgraph.go | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 01083ef7ac..44408acb39 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -39,7 +39,7 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma } // map of module->dependencies + replaces - var dependencies map[string][]string + var dependencies map[string]map[string]struct{} // bidirectional map of module->module name or dependency->dependency var dependencyNames *bimap.BiMap[string, string] @@ -56,8 +56,7 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma // build the dependency graph if typeOfDependency == "module" { for _, module := range parsedWorkFile.Use { - moduleDeps := dependencies[module.Path] - for _, dep := range moduleDeps { + for dep, _ := range dependencies[module.Path] { // check if the full package name (e.g. github.com/myorg/myrepo/mymodule) is in the list of modules. If it is, add it as a dependency after renaming renamedDep, hasDep := dependencyNames.GetInverse(dep) if hasDep { @@ -90,10 +89,8 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma for _, packageInModule := range allPackagesInModule { renamedPackage, hasPackage := dependencyNames.Get(packageInModule) - t := dependencies[renamedPackage] - if hasPackage { - for _, dep := range t { + for dep, _ := range dependencies[renamedPackage] { dep = strings.TrimPrefix(dep, `"`) dep = strings.TrimSuffix(dep, `"`) @@ -139,10 +136,10 @@ func extractGoFileNames(pwd string, currentModule string, currentPackage string, } // makeDepMaps makes a dependency map and a bidirectional map of dep<->module. -func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) (dependencies map[string][]string, dependencyNames *bimap.BiMap[string, string], packagesPerModule map[string][]string, err error) { +func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) (dependencies map[string]map[string]struct{}, dependencyNames *bimap.BiMap[string, string], packagesPerModule map[string][]string, err error) { // map of module->dependencies + replaces // map of packages -> dependencies - dependencies = make(map[string][]string) + dependencies = make(map[string]map[string]struct{}) // bidirectional map of module->module name // bidirectional map of package->package name, relative to public names. dependencyNames = bimap.NewBiMap[string, string]() @@ -167,14 +164,14 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } dependencyNames.Insert(module.Path, parsedModFile.Module.Mod.Path) - dependencies[module.Path] = make([]string, 0) + dependencies[module.Path] = make(map[string]struct{}) // include all requires and replaces, as they are dependencies for _, require := range parsedModFile.Require { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.Mod.Path)) + dependencies[module.Path][convertRelPath(repoPath, module.Path, require.Mod.Path)] = struct{}{} } for _, require := range parsedModFile.Replace { - dependencies[module.Path] = append(dependencies[module.Path], convertRelPath(repoPath, module.Path, require.New.Path)) + dependencies[module.Path][convertRelPath(repoPath,module.Path, require.New.Path)] = struct{}{} } } } @@ -219,15 +216,16 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) for _, module := range uses { for _, packageInModule := range packagesPerModule[module.Path] { + publicPackageName, _ := dependencyNames.Get(packageInModule) + dependencies[publicPackageName] = make(map[string]struct{}) for _, file := range extractedGoFileNames[module.Path[1:]][packageInModule] { fset := token.NewFileSet() f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) if err != nil { } - publicPackageName, _ := dependencyNames.Get(packageInModule) for _, s := range f.Imports { - dependencies[publicPackageName] = append(dependencies[publicPackageName], s.Path.Value) + dependencies[publicPackageName][s.Path.Value] = struct{}{} } } } From 94dd773c7e5a7fbddb1235ca5f5459b7c1b31864 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sun, 24 Mar 2024 22:27:02 -0600 Subject: [PATCH 06/39] fix: optimize loop --- .../git-changes-action/detector/depgraph.go | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 44408acb39..7005cb7c82 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -215,25 +215,24 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } for _, module := range uses { - for _, packageInModule := range packagesPerModule[module.Path] { + for packageInModule, files := range extractedGoFileNames[module.Path[1:]] { publicPackageName, _ := dependencyNames.Get(packageInModule) dependencies[publicPackageName] = make(map[string]struct{}) - for _, file := range extractedGoFileNames[module.Path[1:]][packageInModule] { - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) - if err != nil { - } + for _, file := range files { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) - for _, s := range f.Imports { - dependencies[publicPackageName][s.Path.Value] = struct{}{} - } + if err != nil { + } + + for _, s := range f.Imports { + dependencies[publicPackageName][s.Path.Value] = struct{}{} + } } } } } - fmt.Println(dependencies) - return dependencies, dependencyNames, packagesPerModule, nil } From dd43542c44a38e6bb84a3a7cf594ef4291bd1701 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sun, 24 Mar 2024 23:11:50 -0600 Subject: [PATCH 07/39] fix: change naming of packages in extractGoFiles --- .../git-changes-action/detector/depgraph.go | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 7005cb7c82..a06716f76b 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -124,13 +124,7 @@ func extractGoFileNames(pwd string, currentModule string, currentPackage string, extractGoFileNames(pwd + "/" + entry.Name(), currentModule, entry.Name(), goFiles) } else if strings.Contains(entry.Name(), ".go") { fileName := pwd + "/" + entry.Name() - var packageName string - if currentModule == currentPackage { - packageName = "/" + currentPackage - } else { - packageName = "/" + currentModule + "/" + currentPackage - } - goFiles[packageName] = append(goFiles[packageName], fileName) + goFiles["/" + currentPackage] = append(goFiles["/" + currentPackage], fileName) } } } @@ -201,12 +195,19 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) extractGoFileNames(pwd + module.Path[1:], module.Path[2:], module.Path[2:], extractedGoFileNames[module.Path[1:]]) - for relativePackageName, _ := range extractedGoFileNames[module.Path[1:]] { + for packageName, _ := range extractedGoFileNames[module.Path[1:]] { + var relativePackageName string + if strings.Contains(module.Path[1:], packageName) { + relativePackageName = module.Path[1:] + } else { + relativePackageName = module.Path[1:] + packageName + } + var publicPackageName string - if strings.Contains(parsedModFile.Module.Mod.Path, relativePackageName) { + if strings.Contains(parsedModFile.Module.Mod.Path, packageName) { publicPackageName = parsedModFile.Module.Mod.Path } else { - publicPackageName = parsedModFile.Module.Mod.Path + relativePackageName + publicPackageName = parsedModFile.Module.Mod.Path + packageName } packagesPerModule[module.Path] = append(packagesPerModule[module.Path], relativePackageName) @@ -214,9 +215,14 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } } + fmt.Println(dependencyNames.GetForwardMap()) + + + for _, module := range uses { for packageInModule, files := range extractedGoFileNames[module.Path[1:]] { - publicPackageName, _ := dependencyNames.Get(packageInModule) + publicPackageName, _ := dependencyNames.Get(module.Path[1:] + packageInModule) + dependencies[publicPackageName] = make(map[string]struct{}) for _, file := range files { fset := token.NewFileSet() From 6f4ceb27dad9fb56911868245b53f3ba326a2829 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:13:26 -0600 Subject: [PATCH 08/39] fix: package naming nested packages whose directory name is found in the module's name, were overriding the module's dependencyNames entry, this commit fixes that issue --- .../git-changes-action/detector/depgraph.go | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index a06716f76b..8523c0aed6 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -121,10 +121,16 @@ func extractGoFileNames(pwd string, currentModule string, currentPackage string, for _, entry := range ls { if entry.IsDir() { - extractGoFileNames(pwd + "/" + entry.Name(), currentModule, entry.Name(), goFiles) + extractGoFileNames(pwd + "/" + entry.Name(), currentModule + "/" + entry.Name(), entry.Name(), goFiles) } else if strings.Contains(entry.Name(), ".go") { fileName := pwd + "/" + entry.Name() - goFiles["/" + currentPackage] = append(goFiles["/" + currentPackage], fileName) + var packageName string + if currentModule == "" { + packageName = "/" + currentPackage + } else { + packageName = currentModule + } + goFiles[packageName] = append(goFiles[packageName], fileName) } } } @@ -193,18 +199,19 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - extractGoFileNames(pwd + module.Path[1:], module.Path[2:], module.Path[2:], extractedGoFileNames[module.Path[1:]]) + + extractGoFileNames(pwd + module.Path[1:], "", module.Path[2:], extractedGoFileNames[module.Path[1:]]) for packageName, _ := range extractedGoFileNames[module.Path[1:]] { var relativePackageName string - if strings.Contains(module.Path[1:], packageName) { + if strings.HasSuffix(module.Path[1:], packageName) { relativePackageName = module.Path[1:] } else { relativePackageName = module.Path[1:] + packageName } var publicPackageName string - if strings.Contains(parsedModFile.Module.Mod.Path, packageName) { + if strings.HasSuffix(parsedModFile.Module.Mod.Path, packageName) { publicPackageName = parsedModFile.Module.Mod.Path } else { publicPackageName = parsedModFile.Module.Mod.Path + packageName @@ -215,10 +222,6 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } } - fmt.Println(dependencyNames.GetForwardMap()) - - - for _, module := range uses { for packageInModule, files := range extractedGoFileNames[module.Path[1:]] { publicPackageName, _ := dependencyNames.Get(module.Path[1:] + packageInModule) @@ -232,7 +235,11 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } for _, s := range f.Imports { + _, hasDep := dependencyNames.GetInverse(s.Path.Value[1:len(s.Path.Value)-1]) + + if hasDep { dependencies[publicPackageName][s.Path.Value] = struct{}{} + } } } } From fad4ff6948088f16571e7684054aea915ced59f5 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:05:16 -0600 Subject: [PATCH 09/39] refactor: remove dependency filtering from graph creation dependency filtering is now taken care of on dependency map creation --- .../git-changes-action/detector/depgraph.go | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 8523c0aed6..2b92f30d12 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -85,27 +85,16 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma if typeOfDependency == "packages" { for _, module := range parsedWorkFile.Use { - allPackagesInModule := packagesPerModule[module.Path] - - for _, packageInModule := range allPackagesInModule { - renamedPackage, hasPackage := dependencyNames.Get(packageInModule) - if hasPackage { - for dep, _ := range dependencies[renamedPackage] { - dep = strings.TrimPrefix(dep, `"`) - dep = strings.TrimSuffix(dep, `"`) - - renamedDep, hasDep := dependencyNames.GetInverse(dep) - if hasDep { - err = depGraph.DependOn(packageInModule, renamedDep) - if err != nil { - fmt.Println("THERE IS AN ERROR", err, packageInModule, renamedDep) - } + for _, relativePackageName := range packagesPerModule[module.Path] { + for relativePackageDependencyName, _ := range dependencies[relativePackageName] { + err = depGraph.DependOn(relativePackageName, relativePackageDependencyName) + if err != nil { + fmt.Println("THERE IS AN ERROR", err, relativePackageName, relativePackageDependencyName) } } - } - for dep := range depGraph.Dependencies(packageInModule) { - moduleDeps[packageInModule] = append(moduleDeps[packageInModule], dep) + for dep := range depGraph.Dependencies(relativePackageName) { + moduleDeps[relativePackageName] = append(moduleDeps[relativePackageName], dep) } } } @@ -224,9 +213,9 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) for _, module := range uses { for packageInModule, files := range extractedGoFileNames[module.Path[1:]] { - publicPackageName, _ := dependencyNames.Get(module.Path[1:] + packageInModule) + relativePackaeName := module.Path[1:] + packageInModule - dependencies[publicPackageName] = make(map[string]struct{}) + dependencies[relativePackaeName] = make(map[string]struct{}) for _, file := range files { fset := token.NewFileSet() f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) @@ -235,10 +224,10 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } for _, s := range f.Imports { - _, hasDep := dependencyNames.GetInverse(s.Path.Value[1:len(s.Path.Value)-1]) + renamedDep, hasDep := dependencyNames.GetInverse(s.Path.Value[1:len(s.Path.Value)-1]) if hasDep { - dependencies[publicPackageName][s.Path.Value] = struct{}{} + dependencies[relativePackaeName][renamedDep] = struct{}{} } } } From 5acc601e47f280e8146cdcdb58fe399ae358efd6 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:58:24 -0600 Subject: [PATCH 10/39] fix: package naming in dependencies map --- contrib/git-changes-action/detector/depgraph.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 2b92f30d12..21b867e729 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -188,7 +188,6 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - extractGoFileNames(pwd + module.Path[1:], "", module.Path[2:], extractedGoFileNames[module.Path[1:]]) for packageName, _ := range extractedGoFileNames[module.Path[1:]] { @@ -213,9 +212,14 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) for _, module := range uses { for packageInModule, files := range extractedGoFileNames[module.Path[1:]] { - relativePackaeName := module.Path[1:] + packageInModule + var relativePackageName string + if strings.HasSuffix(module.Path[1:], packageInModule) { + relativePackageName = module.Path[1:] + } else { + relativePackageName = module.Path[1:] + packageInModule + } - dependencies[relativePackaeName] = make(map[string]struct{}) + dependencies[relativePackageName] = make(map[string]struct{}) for _, file := range files { fset := token.NewFileSet() f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) @@ -227,7 +231,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) renamedDep, hasDep := dependencyNames.GetInverse(s.Path.Value[1:len(s.Path.Value)-1]) if hasDep { - dependencies[relativePackaeName][renamedDep] = struct{}{} + dependencies[relativePackageName][renamedDep] = struct{}{} } } } From a843b5e73a18ccb4d9f06a7a53cd102b0fd807dd Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Mon, 25 Mar 2024 19:08:46 -0600 Subject: [PATCH 11/39] fix: add '.' to relative package name construction better dx when default module path is not trimmed every time its used --- .../git-changes-action/detector/depgraph.go | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 21b867e729..b6c8c81069 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -176,7 +176,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) for _, module := range uses { - extractedGoFileNames[module.Path[1:]] = make(map[string][]string) + extractedGoFileNames[module.Path] = make(map[string][]string) modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) if err != nil { @@ -188,14 +188,14 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - extractGoFileNames(pwd + module.Path[1:], "", module.Path[2:], extractedGoFileNames[module.Path[1:]]) + extractGoFileNames(pwd + module.Path[1:], "", module.Path[2:], extractedGoFileNames[module.Path]) - for packageName, _ := range extractedGoFileNames[module.Path[1:]] { + for packageName, _ := range extractedGoFileNames[module.Path] { var relativePackageName string - if strings.HasSuffix(module.Path[1:], packageName) { - relativePackageName = module.Path[1:] + if strings.HasSuffix(module.Path, packageName) { + relativePackageName = module.Path } else { - relativePackageName = module.Path[1:] + packageName + relativePackageName = module.Path + packageName } var publicPackageName string @@ -211,12 +211,12 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } for _, module := range uses { - for packageInModule, files := range extractedGoFileNames[module.Path[1:]] { + for packageInModule, files := range extractedGoFileNames[module.Path] { var relativePackageName string - if strings.HasSuffix(module.Path[1:], packageInModule) { - relativePackageName = module.Path[1:] + if strings.HasSuffix(module.Path, packageInModule) { + relativePackageName = module.Path } else { - relativePackageName = module.Path[1:] + packageInModule + relativePackageName = module.Path + packageInModule } dependencies[relativePackageName] = make(map[string]struct{}) @@ -228,6 +228,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } for _, s := range f.Imports { + // s.Path.Value contains double quotation marks that must be removed before indexing dependencyNames renamedDep, hasDep := dependencyNames.GetInverse(s.Path.Value[1:len(s.Path.Value)-1]) if hasDep { From cdb1b50f84c5874b5bc7b0967fee8202acfa7e38 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Mon, 25 Mar 2024 23:03:56 -0600 Subject: [PATCH 12/39] fix: move recursive cache check to step one --- contrib/git-changes-action/detector/detector.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index 8cbdd3161c..dbebdc8f14 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -21,15 +21,15 @@ import ( func identifyNestedDependencyChange(packageName string, depGraph map[string][]string, ct tree.Tree, packages map[string]bool) (changed bool) { + if _, ok := packages[packageName]; ok { + return packages[packageName] + } + if ct.HasPath(packageName) { packages[packageName] = true return true } - if _, ok := packages[packageName]; ok { - return packages[packageName] - } - deps := depGraph[packageName] changed = false @@ -55,7 +55,7 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO if !common.FileExist(goWorkPath) { return nil, fmt.Errorf("go.work file not found in %s", repoPath) - } + } //nolint: gosec workFile, err := os.ReadFile(goWorkPath) From dba2a8ca25991b557f2d83d1540510e1d87c7a3f Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Mon, 25 Mar 2024 23:05:57 -0600 Subject: [PATCH 13/39] add: filter self-referential dependencies self referentials come from test packages. Also added some comments --- .../git-changes-action/detector/depgraph.go | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index b6c8c81069..1a726df2cc 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -126,20 +126,24 @@ func extractGoFileNames(pwd string, currentModule string, currentPackage string, // makeDepMaps makes a dependency map and a bidirectional map of dep<->module. func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) (dependencies map[string]map[string]struct{}, dependencyNames *bimap.BiMap[string, string], packagesPerModule map[string][]string, err error) { - // map of module->dependencies + replaces + // Can be either: + // map of module->depndencies // map of packages -> dependencies + // depends on typeOfDependency dependencies = make(map[string]map[string]struct{}) + // bidirectional map of module->module name - // bidirectional map of package->package name, relative to public names. + // bidirectional map of package->package name + // Maps relative to public names, used to filer out all external libraries/packages. dependencyNames = bimap.NewBiMap[string, string]() + // map of module->packages packagesPerModule = make(map[string][]string) // iterate through each module in the go.work file - // create a list of dependencies for each module based on modules or packages - // and module names or package names + // 1. Create a list of dependencies for each module + // 2. Map public names to private names for each module //nolint: gose - if typeOfDependency == "module" { for _, module := range uses { modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) @@ -166,7 +170,6 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } - if typeOfDependency == "packages" { extractedGoFileNames := make(map[string]map[string][]string) @@ -175,6 +178,10 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } + // iterate through each module in the go.work file + // 1. Extract all go files filepaths for each package. + // 2. Create a map where key is module, value is an array with all packages in the module + // 3. Map public name to relative name for each package (used to filter external library/package imports) for _, module := range uses { extractedGoFileNames[module.Path] = make(map[string][]string) @@ -210,6 +217,10 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) } } + // iterate through each module in the go.work file + // For every package in the module + // using the filepaths extracted on the previous loop, parse files and extract imports. + // Ignore any external library/package imports for _, module := range uses { for packageInModule, files := range extractedGoFileNames[module.Path] { var relativePackageName string @@ -231,7 +242,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) // s.Path.Value contains double quotation marks that must be removed before indexing dependencyNames renamedDep, hasDep := dependencyNames.GetInverse(s.Path.Value[1:len(s.Path.Value)-1]) - if hasDep { + if hasDep && (relativePackageName != renamedDep){ dependencies[relativePackageName][renamedDep] = struct{}{} } } From 521a0238334134bd76d06685a2ad550910404c95 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Tue, 26 Mar 2024 00:54:45 -0600 Subject: [PATCH 14/39] add: graceful error handling --- .../git-changes-action/detector/depgraph.go | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 1a726df2cc..dc188e0451 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -47,14 +47,13 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma // create a list of dependencies for each module // and module names dependencies, dependencyNames, packagesPerModule, err = makeDepMaps(repoPath, parsedWorkFile.Use, typeOfDependency) - if err != nil { return nil, nil, fmt.Errorf("failed to create dependency maps: %w", err) } depGraph := depgraph.New() // build the dependency graph - if typeOfDependency == "module" { + if typeOfDependency == "modules" { for _, module := range parsedWorkFile.Use { for dep, _ := range dependencies[module.Path] { // check if the full package name (e.g. github.com/myorg/myrepo/mymodule) is in the list of modules. If it is, add it as a dependency after renaming @@ -88,8 +87,9 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma for _, relativePackageName := range packagesPerModule[module.Path] { for relativePackageDependencyName, _ := range dependencies[relativePackageName] { err = depGraph.DependOn(relativePackageName, relativePackageDependencyName) - if err != nil { - fmt.Println("THERE IS AN ERROR", err, relativePackageName, relativePackageDependencyName) + // Circular dependencies are fine as long as both packages are in the same module + if err != nil && !(strings.Contains(relativePackageDependencyName, module.Path) && strings.Contains(relativePackageName, module.Path)){ + return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", relativePackageName, relativePackageDependencyName, err) } } @@ -103,9 +103,10 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma return moduleDeps, packagesPerModule, nil } -func extractGoFileNames(pwd string, currentModule string, currentPackage string, goFiles map[string][]string) { +func extractGoFileNames(pwd string, currentModule string, currentPackage string, goFiles map[string][]string) (err error) { ls, err := os.ReadDir(pwd) if err != nil { + return err } for _, entry := range ls { @@ -122,6 +123,8 @@ func extractGoFileNames(pwd string, currentModule string, currentPackage string, goFiles[packageName] = append(goFiles[packageName], fileName) } } + + return nil } // makeDepMaps makes a dependency map and a bidirectional map of dep<->module. @@ -144,7 +147,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) // 1. Create a list of dependencies for each module // 2. Map public names to private names for each module //nolint: gose - if typeOfDependency == "module" { + if typeOfDependency == "modules" { for _, module := range uses { modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) if err != nil { @@ -175,6 +178,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) pwd, err := os.Getwd() if err != nil { + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("Failed to read current directory.", err) } @@ -195,7 +199,11 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - extractGoFileNames(pwd + module.Path[1:], "", module.Path[2:], extractedGoFileNames[module.Path]) + // module.Path = ./moduleName + err = extractGoFileNames(pwd + module.Path[1:], "", module.Path[2:], extractedGoFileNames[module.Path]) + if err != nil { + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to extract go files for module %s: %w", module.Path, err) + } for packageName, _ := range extractedGoFileNames[module.Path] { var relativePackageName string @@ -236,6 +244,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) if err != nil { + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse go file %s in package %s: %w", file, relativePackageName, err) } for _, s := range f.Imports { From 6f10f0946546e0ed33abd91ebf41488762f55146 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Tue, 26 Mar 2024 01:01:58 -0600 Subject: [PATCH 15/39] fix: swtich from contains to hasSuffix This will avoid including .goreleaser files --- contrib/git-changes-action/detector/depgraph.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index dc188e0451..07ecf1c3d9 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -112,7 +112,7 @@ func extractGoFileNames(pwd string, currentModule string, currentPackage string, for _, entry := range ls { if entry.IsDir() { extractGoFileNames(pwd + "/" + entry.Name(), currentModule + "/" + entry.Name(), entry.Name(), goFiles) - } else if strings.Contains(entry.Name(), ".go") { + } else if strings.HasSuffix(entry.Name(), ".go") { fileName := pwd + "/" + entry.Name() var packageName string if currentModule == "" { From 86813b0b96e717e5f27108606e36a2ffc72fc1ca Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:54:33 -0600 Subject: [PATCH 16/39] fix: if condition --- contrib/git-changes-action/detector/detector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index dbebdc8f14..5760eac986 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -37,7 +37,7 @@ func identifyNestedDependencyChange(packageName string, depGraph map[string][]st for _, dep := range deps { changed = identifyNestedDependencyChange(dep, depGraph, ct, packages) - if changed == true { + if changed { break } } From 77f5308d29a3a743a06a8dbbb19e055fdd87e435 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:27:49 -0600 Subject: [PATCH 17/39] fix: remove identifyNestedDependencyChange degraph library already does this work, dependencies are transitive. --- .../git-changes-action/detector/depgraph.go | 10 +++-- .../git-changes-action/detector/detector.go | 43 ++++++------------- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 07ecf1c3d9..67617053f2 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -92,12 +92,16 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", relativePackageName, relativePackageDependencyName, err) } } + } + } - for dep := range depGraph.Dependencies(relativePackageName) { - moduleDeps[relativePackageName] = append(moduleDeps[relativePackageName], dep) + for _, module := range parsedWorkFile.Use { + for _, relativePackageName := range packagesPerModule[module.Path] { + for dep := range depGraph.Dependencies(relativePackageName) { + moduleDeps[relativePackageName] = append(moduleDeps[relativePackageName], dep) + } } } - } } return moduleDeps, packagesPerModule, nil diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index 5760eac986..a753588ad9 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -19,33 +19,6 @@ import ( "strings" ) - -func identifyNestedDependencyChange(packageName string, depGraph map[string][]string, ct tree.Tree, packages map[string]bool) (changed bool) { - if _, ok := packages[packageName]; ok { - return packages[packageName] - } - - if ct.HasPath(packageName) { - packages[packageName] = true - return true - } - - deps := depGraph[packageName] - changed = false - - if len(deps) != 0 { - for _, dep := range deps { - changed = identifyNestedDependencyChange(dep, depGraph, ct, packages) - - if changed { - break - } - } - } - - packages[packageName] = changed - return changed -} // DetectChangedModules is the change detector client. // nolint: cyclop func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeOfDependency string) (modules map[string]bool, err error) { @@ -112,10 +85,18 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO if includeDeps { for _, packageName := range packagesPerModule[module.Path] { - changed = identifyNestedDependencyChange(packageName, depGraph, ct, packages) - - if changed == true { - break + for _, dep := range depGraph[packageName] { + if ct.HasPath(dep) { + changed = true + } + + if changed { + break + } + } + + if changed { + break } } } From b85315f12d1852fc3144b869583e347a87daf0d8 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:46:50 -0600 Subject: [PATCH 18/39] fix: remove caching dependency resolution is no longer recursive --- contrib/git-changes-action/detector/detector.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index a753588ad9..849a787569 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -67,19 +67,12 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO } if typeOfDependency == "packages" { - // This map stores package change identification - // map(package->isChanged bool) - // Reduces reduntant recursion over packages whose dependency tree has been prevoziously analyzed. - // Essentially caching recursion, so if package was found and analyzed through another package's dependency tree, it is not analyzed again. - var packages = make(map[string]bool) - for _, module := range parsedWorkFile.Use { changed := false if ct.HasPath(module.Path) { changed = true modules[module.Path] = changed - packages[module.Path] = changed continue } @@ -101,10 +94,8 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO } } - packages[module.Path] = changed modules[module.Path] = changed } - } return modules, nil From c7fc018a534146d03ae5d21aa3f0d2f4e3002a6d Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:51:00 -0600 Subject: [PATCH 19/39] fix: clean loop --- .../git-changes-action/detector/detector.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index 849a787569..acfcca16c7 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -78,16 +78,20 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO if includeDeps { for _, packageName := range packagesPerModule[module.Path] { - for _, dep := range depGraph[packageName] { - if ct.HasPath(dep) { - changed = true + if ct.HasPath(packageName) { + changed = true + } else { + for _, dep := range depGraph[packageName] { + if ct.HasPath(dep) { + changed = true + break + } } - - if changed { - break - } } + // If a package is flagged as changed + // its not necessary to analyze the rest, + // the module can be flagged as changed if changed { break } From f43e9ceddad81648ceb511336ebd68c41b7260cf Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:53:55 -0600 Subject: [PATCH 20/39] add: comments --- contrib/git-changes-action/detector/detector.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index acfcca16c7..bd3809338e 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -83,6 +83,9 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO } else { for _, dep := range depGraph[packageName] { if ct.HasPath(dep) { + // If a dependency is flagged as changed + // its not necessary to analyze the remaining dependences, + // the package can be flagged as changed. changed = true break } From bbd39669464c93964702348fdfc44113b79f6145 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:17:38 -0600 Subject: [PATCH 21/39] fix: add formatting directive --- contrib/git-changes-action/detector/depgraph.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index 67617053f2..cd5b170a7a 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -182,7 +182,7 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) pwd, err := os.Getwd() if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("Failed to read current directory.", err) + return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("Failed to read current directory: %w", err) } From 89c71270dd0d0768a4c67224fd57f252d444647e Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:17:55 -0600 Subject: [PATCH 22/39] fix tests --- contrib/git-changes-action/detector/detector_test.go | 6 +++--- contrib/git-changes-action/detector/export_test.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/git-changes-action/detector/detector_test.go b/contrib/git-changes-action/detector/detector_test.go index 5093168102..e82c5e3d76 100644 --- a/contrib/git-changes-action/detector/detector_test.go +++ b/contrib/git-changes-action/detector/detector_test.go @@ -23,10 +23,10 @@ func (d *DetectorSuite) TestChangedModules() { ct, err := detector.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") Nil(d.T(), err, "should not return an error") - withDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, true) + withDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, true, "modules") Nil(d.T(), err, "should not return an error") - withoutDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, false) + withoutDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, false, "modules") Nil(d.T(), err, "should not return an error") False(d.T(), withoutDeps["./cmd/app1"]) @@ -41,7 +41,7 @@ func (d *DetectorSuite) TestChangedModules() { } func (d *DetectorSuite) TestGetDependencyDag() { - deps, err := detector.GetDependencyDag(d.sourceRepo.dir) + deps, _, err := detector.GetDependencyDag(d.sourceRepo.dir) Nil(d.T(), err, "should not return an error") Equal(d.T(), deps["./cmd/app1"], []string{"./lib"}) diff --git a/contrib/git-changes-action/detector/export_test.go b/contrib/git-changes-action/detector/export_test.go index 40295fd6c2..4d600af704 100644 --- a/contrib/git-changes-action/detector/export_test.go +++ b/contrib/git-changes-action/detector/export_test.go @@ -5,8 +5,8 @@ import ( "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/actionscore" ) -func GetDependencyDag(repoPath string) (map[string][]string, error) { - return getDependencyGraph(repoPath) +func GetDependencyDag(repoPath string) (map[string][]string, map[string][]string, error) { + return getDependencyGraph(repoPath, "modules") } func GetHead(repo *git.Repository, ghContext *actionscore.Context, head string) (string, error) { From 7e9af019a0723ec924218904c892fec5a262a916 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:32:11 -0600 Subject: [PATCH 23/39] fix: clean implementation --- .../git-changes-action/detector/detector.go | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index bd3809338e..26a0b69cf5 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -78,19 +78,7 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO if includeDeps { for _, packageName := range packagesPerModule[module.Path] { - if ct.HasPath(packageName) { - changed = true - } else { - for _, dep := range depGraph[packageName] { - if ct.HasPath(dep) { - // If a dependency is flagged as changed - // its not necessary to analyze the remaining dependences, - // the package can be flagged as changed. - changed = true - break - } - } - } + changed = checkPackageDependencies(packageName, ct, depGraph) // If a package is flagged as changed // its not necessary to analyze the rest, @@ -108,6 +96,19 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO return modules, nil } +func checkPackageDependencies(packageName string, ct tree.Tree, depGraph map[string][]string) bool { + if ct.HasPath(packageName) { + return true + } + for _, dep := range depGraph[packageName] { + if ct.HasPath(dep) { + return true + } + } + return false +} + + // getChangeTreeFromGit returns a tree of all the files that have changed between the current commit and the commit with the given hash. // nolint: cyclop, gocognit func getChangeTreeFromGit(repoPath string, ghContext *actionscore.Context, head, base string) (tree.Tree, error) { From a8cb54047f212eef7827d1d3120c46eac5c1e021 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:46:38 -0600 Subject: [PATCH 24/39] fix: improve readability --- contrib/git-changes-action/detector/detector.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index 26a0b69cf5..b8eef924c6 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -78,13 +78,12 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO if includeDeps { for _, packageName := range packagesPerModule[module.Path] { - changed = checkPackageDependencies(packageName, ct, depGraph) - - // If a package is flagged as changed - // its not necessary to analyze the rest, - // the module can be flagged as changed - if changed { - break + if isPackageChanged(packageName, ct, depGraph) { + changed = true + // If a package is flagged as changed + // its not necessary to analyze the remaining packages, + // module can be flagged as changed + break } } } @@ -96,10 +95,11 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO return modules, nil } -func checkPackageDependencies(packageName string, ct tree.Tree, depGraph map[string][]string) bool { +func isPackageChanged(packageName string, ct tree.Tree, depGraph map[string][]string) bool { if ct.HasPath(packageName) { return true } + for _, dep := range depGraph[packageName] { if ct.HasPath(dep) { return true From d15cfe703dae9653dfc942c33475272e39e9b2bf Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:00:58 -0600 Subject: [PATCH 25/39] improve readability --- contrib/git-changes-action/detector/detector.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/detector.go index b8eef924c6..bd5c0bbd69 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/detector.go @@ -72,11 +72,9 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeO if ct.HasPath(module.Path) { changed = true - modules[module.Path] = changed - continue } - if includeDeps { + if includeDeps && !changed { for _, packageName := range packagesPerModule[module.Path] { if isPackageChanged(packageName, ct, depGraph) { changed = true From 73f4eeb4bfbf4e520c53700df76f8161ebb20007 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Thu, 28 Mar 2024 09:59:16 -0600 Subject: [PATCH 26/39] fix: clean recursion previous implementation polluted the callstack, this commit implements loop based file extraction --- .../git-changes-action/detector/depgraph.go | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go index cd5b170a7a..ff5176e9ab 100644 --- a/contrib/git-changes-action/detector/depgraph.go +++ b/contrib/git-changes-action/detector/depgraph.go @@ -107,27 +107,36 @@ func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps ma return moduleDeps, packagesPerModule, nil } -func extractGoFileNames(pwd string, currentModule string, currentPackage string, goFiles map[string][]string) (err error) { - ls, err := os.ReadDir(pwd) - if err != nil { - return err - } +func extractGoFileNames(pwd string, currentPackage string, goFiles map[string][]string) (err error) { + + searchNext := make(map[string]string) + _, packageDir := path.Split(currentPackage) + searchNext[pwd] = packageDir + + for len(searchNext) > 0 { + discovered := make(map[string]string) + for path, dirName := range searchNext { + err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() && !(path == filePath) { + discovered[filePath] = info.Name() + return filepath.SkipDir + } else if strings.HasSuffix(info.Name(), ".go") { + goFiles["/" + dirName] = append(goFiles["/" + dirName], filePath) + } + + return nil + }) - for _, entry := range ls { - if entry.IsDir() { - extractGoFileNames(pwd + "/" + entry.Name(), currentModule + "/" + entry.Name(), entry.Name(), goFiles) - } else if strings.HasSuffix(entry.Name(), ".go") { - fileName := pwd + "/" + entry.Name() - var packageName string - if currentModule == "" { - packageName = "/" + currentPackage - } else { - packageName = currentModule + if err != nil { + fmt.Println("Error walking the path: ", err) } - goFiles[packageName] = append(goFiles[packageName], fileName) } + searchNext = discovered } - return nil } @@ -191,7 +200,6 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) // 2. Create a map where key is module, value is an array with all packages in the module // 3. Map public name to relative name for each package (used to filter external library/package imports) for _, module := range uses { - extractedGoFileNames[module.Path] = make(map[string][]string) modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) if err != nil { @@ -203,8 +211,8 @@ func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) } - // module.Path = ./moduleName - err = extractGoFileNames(pwd + module.Path[1:], "", module.Path[2:], extractedGoFileNames[module.Path]) + extractedGoFileNames[module.Path] = make(map[string][]string) + err = extractGoFileNames(pwd + module.Path[1:], module.Path[1:], extractedGoFileNames[module.Path]) if err != nil { return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to extract go files for module %s: %w", module.Path, err) } From 3793f2e634fa56d2eecd88417aad0807951a30fa Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:31:00 -0600 Subject: [PATCH 27/39] refactor: split detector package into three 1. git - contains all git related utilities 2. packagedetector - contains logic to detect changes based on package level dependencies 3. moduledetector - contains logic to detect changes based on module level dependencies --- .../git-changes-action/detector/depgraph.go | 301 ------------------ contrib/git-changes-action/detector/doc.go | 2 - .../detector/export_test.go | 14 - .../git-changes-action/detector/git/doc.go | 2 + .../detector/{ => git}/eventtype_string.go | 4 +- .../detector/{detector.go => git/git.go} | 99 +----- .../detector/{ => git}/ref.go | 2 +- .../detector/module/depgraph.go | 148 +++++++++ .../detector/module/detector.go | 59 ++++ .../detector/{ => module}/detector_test.go | 22 +- .../git-changes-action/detector/module/doc.go | 3 + .../detector/module/export_test.go | 5 + .../detector/{ => module}/suite_test.go | 2 +- .../detector/package/depgraph.go | 202 ++++++++++++ .../detector/package/detector.go | 76 +++++ .../detector/package/doc.go | 3 + contrib/git-changes-action/main.go | 16 +- 17 files changed, 527 insertions(+), 433 deletions(-) delete mode 100644 contrib/git-changes-action/detector/depgraph.go delete mode 100644 contrib/git-changes-action/detector/doc.go delete mode 100644 contrib/git-changes-action/detector/export_test.go create mode 100644 contrib/git-changes-action/detector/git/doc.go rename contrib/git-changes-action/detector/{ => git}/eventtype_string.go (98%) rename contrib/git-changes-action/detector/{detector.go => git/git.go} (74%) rename contrib/git-changes-action/detector/{ => git}/ref.go (99%) create mode 100644 contrib/git-changes-action/detector/module/depgraph.go create mode 100644 contrib/git-changes-action/detector/module/detector.go rename contrib/git-changes-action/detector/{ => module}/detector_test.go (76%) create mode 100644 contrib/git-changes-action/detector/module/doc.go create mode 100644 contrib/git-changes-action/detector/module/export_test.go rename contrib/git-changes-action/detector/{ => module}/suite_test.go (99%) create mode 100644 contrib/git-changes-action/detector/package/depgraph.go create mode 100644 contrib/git-changes-action/detector/package/detector.go create mode 100644 contrib/git-changes-action/detector/package/doc.go diff --git a/contrib/git-changes-action/detector/depgraph.go b/contrib/git-changes-action/detector/depgraph.go deleted file mode 100644 index ff5176e9ab..0000000000 --- a/contrib/git-changes-action/detector/depgraph.go +++ /dev/null @@ -1,301 +0,0 @@ -package detector - -import ( - "fmt" - "go/parser" - "go/token" - "os" - "path" - "path/filepath" - "strings" - - "github.com/ethereum/go-ethereum/common" - "github.com/kendru/darwin/go/depgraph" - "github.com/vishalkuo/bimap" - "golang.org/x/mod/modfile" -) - -// getDependencyGraph returns a dependency graph of all the modules in the go.work file that refer to other modules in the go.work file -// returns a map of module (./my_module)->(./my_module_dependency1,./my_module_dependency2). -// nolint: cyclop -func getDependencyGraph(repoPath string, typeOfDependency string) (moduleDeps map[string][]string, packagesPerModule map[string][]string, err error) { - moduleDeps = make(map[string][]string) - // parse the go.work file - goWorkPath := path.Join(repoPath, "go.work") - - if !common.FileExist(goWorkPath) { - return nil, nil, fmt.Errorf("go.work file not found in %s", repoPath) - } - - //nolint: gosec - workFile, err := os.ReadFile(goWorkPath) - if err != nil { - return nil, nil, fmt.Errorf("failed to read go.work file: %w", err) - } - - parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse go.work file: %w", err) - } - - // map of module->dependencies + replaces - var dependencies map[string]map[string]struct{} - // bidirectional map of module->module name or dependency->dependency - var dependencyNames *bimap.BiMap[string, string] - - // iterate through each module in the go.work file - // create a list of dependencies for each module - // and module names - dependencies, dependencyNames, packagesPerModule, err = makeDepMaps(repoPath, parsedWorkFile.Use, typeOfDependency) - if err != nil { - return nil, nil, fmt.Errorf("failed to create dependency maps: %w", err) - } - - depGraph := depgraph.New() - // build the dependency graph - if typeOfDependency == "modules" { - for _, module := range parsedWorkFile.Use { - for dep, _ := range dependencies[module.Path] { - // check if the full package name (e.g. github.com/myorg/myrepo/mymodule) is in the list of modules. If it is, add it as a dependency after renaming - renamedDep, hasDep := dependencyNames.GetInverse(dep) - if hasDep { - err = depGraph.DependOn(module.Path, renamedDep) - if err != nil { - return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) - } - } - - if isRelativeDep(dep) { - // if the dependency is relative, add it as a dependency - err = depGraph.DependOn(module.Path, dep) - if err != nil { - return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) - } - } - } - } - - for _, module := range parsedWorkFile.Use { - for dep := range depGraph.Dependencies(module.Path) { - moduleDeps[module.Path] = append(moduleDeps[module.Path], dep) - } - } - } - - if typeOfDependency == "packages" { - for _, module := range parsedWorkFile.Use { - for _, relativePackageName := range packagesPerModule[module.Path] { - for relativePackageDependencyName, _ := range dependencies[relativePackageName] { - err = depGraph.DependOn(relativePackageName, relativePackageDependencyName) - // Circular dependencies are fine as long as both packages are in the same module - if err != nil && !(strings.Contains(relativePackageDependencyName, module.Path) && strings.Contains(relativePackageName, module.Path)){ - return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", relativePackageName, relativePackageDependencyName, err) - } - } - } - } - - for _, module := range parsedWorkFile.Use { - for _, relativePackageName := range packagesPerModule[module.Path] { - for dep := range depGraph.Dependencies(relativePackageName) { - moduleDeps[relativePackageName] = append(moduleDeps[relativePackageName], dep) - } - } - } - } - - return moduleDeps, packagesPerModule, nil -} - -func extractGoFileNames(pwd string, currentPackage string, goFiles map[string][]string) (err error) { - - searchNext := make(map[string]string) - _, packageDir := path.Split(currentPackage) - searchNext[pwd] = packageDir - - for len(searchNext) > 0 { - discovered := make(map[string]string) - for path, dirName := range searchNext { - err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() && !(path == filePath) { - discovered[filePath] = info.Name() - return filepath.SkipDir - } else if strings.HasSuffix(info.Name(), ".go") { - goFiles["/" + dirName] = append(goFiles["/" + dirName], filePath) - } - - return nil - }) - - if err != nil { - fmt.Println("Error walking the path: ", err) - } - } - searchNext = discovered - } - return nil -} - -// makeDepMaps makes a dependency map and a bidirectional map of dep<->module. -func makeDepMaps(repoPath string, uses []*modfile.Use, typeOfDependency string) (dependencies map[string]map[string]struct{}, dependencyNames *bimap.BiMap[string, string], packagesPerModule map[string][]string, err error) { - // Can be either: - // map of module->depndencies - // map of packages -> dependencies - // depends on typeOfDependency - dependencies = make(map[string]map[string]struct{}) - - // bidirectional map of module->module name - // bidirectional map of package->package name - // Maps relative to public names, used to filer out all external libraries/packages. - dependencyNames = bimap.NewBiMap[string, string]() - - // map of module->packages - packagesPerModule = make(map[string][]string) - - // iterate through each module in the go.work file - // 1. Create a list of dependencies for each module - // 2. Map public names to private names for each module - //nolint: gose - if typeOfDependency == "modules" { - for _, module := range uses { - modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) - } - - parsedModFile, err := modfile.Parse(module.Path, modContents, nil) - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) - } - - dependencyNames.Insert(module.Path, parsedModFile.Module.Mod.Path) - dependencies[module.Path] = make(map[string]struct{}) - - // include all requires and replaces, as they are dependencies - for _, require := range parsedModFile.Require { - dependencies[module.Path][convertRelPath(repoPath, module.Path, require.Mod.Path)] = struct{}{} - } - for _, require := range parsedModFile.Replace { - dependencies[module.Path][convertRelPath(repoPath,module.Path, require.New.Path)] = struct{}{} - } - } - } - - - if typeOfDependency == "packages" { - extractedGoFileNames := make(map[string]map[string][]string) - - pwd, err := os.Getwd() - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("Failed to read current directory: %w", err) - } - - - // iterate through each module in the go.work file - // 1. Extract all go files filepaths for each package. - // 2. Create a map where key is module, value is an array with all packages in the module - // 3. Map public name to relative name for each package (used to filter external library/package imports) - for _, module := range uses { - - modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) - } - - parsedModFile, err := modfile.Parse(module.Path, modContents, nil) - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) - } - - extractedGoFileNames[module.Path] = make(map[string][]string) - err = extractGoFileNames(pwd + module.Path[1:], module.Path[1:], extractedGoFileNames[module.Path]) - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to extract go files for module %s: %w", module.Path, err) - } - - for packageName, _ := range extractedGoFileNames[module.Path] { - var relativePackageName string - if strings.HasSuffix(module.Path, packageName) { - relativePackageName = module.Path - } else { - relativePackageName = module.Path + packageName - } - - var publicPackageName string - if strings.HasSuffix(parsedModFile.Module.Mod.Path, packageName) { - publicPackageName = parsedModFile.Module.Mod.Path - } else { - publicPackageName = parsedModFile.Module.Mod.Path + packageName - } - - packagesPerModule[module.Path] = append(packagesPerModule[module.Path], relativePackageName) - dependencyNames.Insert(relativePackageName, publicPackageName) - } - } - - // iterate through each module in the go.work file - // For every package in the module - // using the filepaths extracted on the previous loop, parse files and extract imports. - // Ignore any external library/package imports - for _, module := range uses { - for packageInModule, files := range extractedGoFileNames[module.Path] { - var relativePackageName string - if strings.HasSuffix(module.Path, packageInModule) { - relativePackageName = module.Path - } else { - relativePackageName = module.Path + packageInModule - } - - dependencies[relativePackageName] = make(map[string]struct{}) - for _, file := range files { - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) - - if err != nil { - return dependencies, dependencyNames, packagesPerModule, fmt.Errorf("failed to parse go file %s in package %s: %w", file, relativePackageName, err) - } - - for _, s := range f.Imports { - // s.Path.Value contains double quotation marks that must be removed before indexing dependencyNames - renamedDep, hasDep := dependencyNames.GetInverse(s.Path.Value[1:len(s.Path.Value)-1]) - - if hasDep && (relativePackageName != renamedDep){ - dependencies[relativePackageName][renamedDep] = struct{}{} - } - } - } - } - } - } - - return dependencies, dependencyNames, packagesPerModule, nil -} - -// isRelativeDep returns true if the dependency is relative to the module (starts with ./ or ../). -func isRelativeDep(path string) bool { - return strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../") -} - -// convertRelPath converts a path relative to a module to a path relative to the repository root. -// it does nothing if the path does not start with ./ or ../. -func convertRelPath(repoPath string, modulePath, dependency string) string { - if !isRelativeDep(dependency) { - return dependency - } - - // repo/./module => repo/module - fullModulePath := filepath.Join(repoPath, modulePath) - // repo/module/../dependency => repo/dependency - fullDependencyPath := filepath.Join(fullModulePath, dependency) - // repo/dependency => dependency - trimmedPath := strings.TrimPrefix(fullDependencyPath, repoPath) - if len(trimmedPath) == 0 { - return "." - } - - return fmt.Sprintf(".%s", trimmedPath) -} diff --git a/contrib/git-changes-action/detector/doc.go b/contrib/git-changes-action/detector/doc.go deleted file mode 100644 index c87af8467c..0000000000 --- a/contrib/git-changes-action/detector/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package detector implements a detector for detecting changes in a git repo. -package detector diff --git a/contrib/git-changes-action/detector/export_test.go b/contrib/git-changes-action/detector/export_test.go deleted file mode 100644 index 4d600af704..0000000000 --- a/contrib/git-changes-action/detector/export_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package detector - -import ( - "github.com/go-git/go-git/v5" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/actionscore" -) - -func GetDependencyDag(repoPath string) (map[string][]string, map[string][]string, error) { - return getDependencyGraph(repoPath, "modules") -} - -func GetHead(repo *git.Repository, ghContext *actionscore.Context, head string) (string, error) { - return getHead(repo, ghContext, head) -} diff --git a/contrib/git-changes-action/detector/git/doc.go b/contrib/git-changes-action/detector/git/doc.go new file mode 100644 index 0000000000..3ce63647eb --- /dev/null +++ b/contrib/git-changes-action/detector/git/doc.go @@ -0,0 +1,2 @@ +// Package git provides utilities to interact with git. +package git diff --git a/contrib/git-changes-action/detector/eventtype_string.go b/contrib/git-changes-action/detector/git/eventtype_string.go similarity index 98% rename from contrib/git-changes-action/detector/eventtype_string.go rename to contrib/git-changes-action/detector/git/eventtype_string.go index 7b7b71978d..975faecc64 100644 --- a/contrib/git-changes-action/detector/eventtype_string.go +++ b/contrib/git-changes-action/detector/git/eventtype_string.go @@ -1,7 +1,5 @@ // Code generated by "stringer -type=EventType -linecomment"; DO NOT EDIT. - -package detector - +package git import "strconv" func _() { diff --git a/contrib/git-changes-action/detector/detector.go b/contrib/git-changes-action/detector/git/git.go similarity index 74% rename from contrib/git-changes-action/detector/detector.go rename to contrib/git-changes-action/detector/git/git.go index bd5c0bbd69..f05a5e2656 100644 --- a/contrib/git-changes-action/detector/detector.go +++ b/contrib/git-changes-action/detector/git/git.go @@ -1,11 +1,10 @@ -package detector +package git import ( "encoding/hex" "encoding/json" "errors" "fmt" - "github.com/ethereum/go-ethereum/common" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" @@ -13,100 +12,10 @@ import ( "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/actionscore" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" "github.com/synapsecns/sanguine/core" - "golang.org/x/mod/modfile" "os" - "path" "strings" ) -// DetectChangedModules is the change detector client. -// nolint: cyclop -func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool, typeOfDependency string) (modules map[string]bool, err error) { - modules = make(map[string]bool) - - goWorkPath := path.Join(repoPath, "go.work") - - if !common.FileExist(goWorkPath) { - return nil, fmt.Errorf("go.work file not found in %s", repoPath) - } - - //nolint: gosec - workFile, err := os.ReadFile(goWorkPath) - if err != nil { - return nil, fmt.Errorf("failed to read go.work file: %w", err) - } - - parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) - if err != nil { - return nil, fmt.Errorf("failed to parse go.work file: %w", err) - } - - depGraph, packagesPerModule, err := getDependencyGraph(repoPath, typeOfDependency) - if err != nil { - return nil, fmt.Errorf("could not get dep graph: %w", err) - } - - if (typeOfDependency == "modules") { - for _, module := range parsedWorkFile.Use { - changed := false - if ct.HasPath(module.Path) { - changed = true - } - - if includeDeps { - deps := depGraph[module.Path] - for _, dep := range deps { - if ct.HasPath(dep) { - changed = true - } - } - } - - modules[module.Path] = changed - } - } - - if typeOfDependency == "packages" { - for _, module := range parsedWorkFile.Use { - changed := false - - if ct.HasPath(module.Path) { - changed = true - } - - if includeDeps && !changed { - for _, packageName := range packagesPerModule[module.Path] { - if isPackageChanged(packageName, ct, depGraph) { - changed = true - // If a package is flagged as changed - // its not necessary to analyze the remaining packages, - // module can be flagged as changed - break - } - } - } - - modules[module.Path] = changed - } - } - - return modules, nil -} - -func isPackageChanged(packageName string, ct tree.Tree, depGraph map[string][]string) bool { - if ct.HasPath(packageName) { - return true - } - - for _, dep := range depGraph[packageName] { - if ct.HasPath(dep) { - return true - } - } - return false -} - - // getChangeTreeFromGit returns a tree of all the files that have changed between the current commit and the commit with the given hash. // nolint: cyclop, gocognit func getChangeTreeFromGit(repoPath string, ghContext *actionscore.Context, head, base string) (tree.Tree, error) { @@ -116,7 +25,7 @@ func getChangeTreeFromGit(repoPath string, ghContext *actionscore.Context, head, return nil, fmt.Errorf("could not open repository %s: %w", repoPath, err) } - head, err = getHead(repository, ghContext, head) + head, err = GetHead(repository, ghContext, head) if err != nil { return nil, fmt.Errorf("could not get head: %w", err) } @@ -312,9 +221,9 @@ func getHeadBase(repo *git.Repository) (head string, base string, err error) { return co.Hash().String(), lastCommit.Hash.String(), nil } -// getHead gets the head of the current branch. +// GetHead gets the head of the current branch. // it attempts to mirror the logic of https://github.com/dorny/paths-filter/blob/0ef5f0d812dc7b631d69e07d2491d70fcebc25c8/src/main.ts#L104 -func getHead(repo *git.Repository, ghContext *actionscore.Context, head string) (string, error) { +func GetHead(repo *git.Repository, ghContext *actionscore.Context, head string) (string, error) { if head != "" { return head, nil } diff --git a/contrib/git-changes-action/detector/ref.go b/contrib/git-changes-action/detector/git/ref.go similarity index 99% rename from contrib/git-changes-action/detector/ref.go rename to contrib/git-changes-action/detector/git/ref.go index 204db93f0a..5c8495177c 100644 --- a/contrib/git-changes-action/detector/ref.go +++ b/contrib/git-changes-action/detector/git/ref.go @@ -1,4 +1,4 @@ -package detector +package git import ( "context" diff --git a/contrib/git-changes-action/detector/module/depgraph.go b/contrib/git-changes-action/detector/module/depgraph.go new file mode 100644 index 0000000000..f4b7d2acac --- /dev/null +++ b/contrib/git-changes-action/detector/module/depgraph.go @@ -0,0 +1,148 @@ +package moduledetector + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/kendru/darwin/go/depgraph" + "github.com/vishalkuo/bimap" + "golang.org/x/mod/modfile" + "os" + "path" + "path/filepath" + "strings" +) + +// getDependencyGraph returns a dependency graph of all the modules in the go.work file that refer to other modules in the go.work file +// returns a map of module (./my_module)->(./my_module_dependency1,./my_module_dependency2). +// nolint: cyclop +func getDependencyGraph(repoPath string) (moduleDeps map[string][]string, err error) { + moduleDeps = make(map[string][]string) + // parse the go.work file + goWorkPath := path.Join(repoPath, "go.work") + + if !common.FileExist(goWorkPath) { + return nil, fmt.Errorf("go.work file not found in %s", repoPath) + } + + //nolint: gosec + workFile, err := os.ReadFile(goWorkPath) + if err != nil { + return nil, fmt.Errorf("failed to read go.work file: %w", err) + } + + parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse go.work file: %w", err) + } + + // map of module->dependencies + replaces + var dependencies map[string]map[string]struct{} + // bidirectional map of module->module + // moduleRelativeName <-> modulePublicName + var dependencyNames *bimap.BiMap[string, string] + + // iterate through each module in the go.work file + // create a list of dependencies for each module + // and module names + dependencies, dependencyNames, err = makeDepMaps(repoPath, parsedWorkFile.Use) + if err != nil { + return nil, fmt.Errorf("failed to create dependency maps: %w", err) + } + + depGraph := depgraph.New() + // build the dependency graph + for _, module := range parsedWorkFile.Use { + for dep := range dependencies[module.Path] { + // check if the full package name (e.g. github.com/myorg/myrepo/mymodule) is in the list of modules. If it is, add it as a dependency after renaming + renamedDep, hasDep := dependencyNames.GetInverse(dep) + if hasDep { + err = depGraph.DependOn(module.Path, renamedDep) + if err != nil { + return nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) + } + } + + if isRelativeDep(dep) { + // if the dependency is relative, add it as a dependency + err = depGraph.DependOn(module.Path, dep) + if err != nil { + return nil, fmt.Errorf("failed to add dependency %s -> %s: %w", module.Path, dep, err) + } + } + } + } + + for _, module := range parsedWorkFile.Use { + for dep := range depGraph.Dependencies(module.Path) { + moduleDeps[module.Path] = append(moduleDeps[module.Path], dep) + } + } + + return moduleDeps, nil +} + +// makeDepMaps builds a +// 1. module->dependency map +// 2. bidirectional map of module<->module (moduleRelativeName to modulePublicName). +func makeDepMaps(repoPath string, uses []*modfile.Use) (dependencies map[string]map[string]struct{}, dependencyNames *bimap.BiMap[string, string], err error) { + // map of module->depndencies + dependencies = make(map[string]map[string]struct{}) + + // bidirectional map of module->module name + // Maps relative to public names, used to filer out all external libraries/packages. + dependencyNames = bimap.NewBiMap[string, string]() + + // iterate through each module in the go.work file + // 1. Create a list of dependencies for each module + // 2. Map public names to private names for each module + //nolint: gosec + for _, module := range uses { + modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) + if err != nil { + return dependencies, dependencyNames, fmt.Errorf("failed to read module file %s: %w", module.Path, err) + } + + parsedModFile, err := modfile.Parse(module.Path, modContents, nil) + if err != nil { + return dependencies, dependencyNames, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) + } + + dependencyNames.Insert(module.Path, parsedModFile.Module.Mod.Path) + dependencies[module.Path] = make(map[string]struct{}) + + // include all requires and replaces, as they are dependencies + for _, require := range parsedModFile.Require { + dependencies[module.Path][convertRelPath(repoPath, module.Path, require.Mod.Path)] = struct{}{} + } + for _, require := range parsedModFile.Replace { + dependencies[module.Path][convertRelPath(repoPath, module.Path, require.New.Path)] = struct{}{} + } + } + + return dependencies, dependencyNames, nil +} + +// isRelativeDep returns true if the dependency is relative to the module (starts with ./ or ../). +func isRelativeDep(path string) bool { + return strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../") +} + +// convertRelPath converts a path relative to a module to a path relative to the repository root. +// it does nothing if the path does not start with ./ or ../. +func convertRelPath(repoPath string, modulePath, dependency string) string { + if !isRelativeDep(dependency) { + return dependency + } + + // repo/./module => repo/module + fullModulePath := filepath.Join(repoPath, modulePath) + // repo/module/../dependency => repo/dependency + fullDependencyPath := filepath.Join(fullModulePath, dependency) + // repo/dependency => dependency + trimmedPath := strings.TrimPrefix(fullDependencyPath, repoPath) + if len(trimmedPath) == 0 { + return "." + } + + return fmt.Sprintf(".%s", trimmedPath) +} diff --git a/contrib/git-changes-action/detector/module/detector.go b/contrib/git-changes-action/detector/module/detector.go new file mode 100644 index 0000000000..1873744339 --- /dev/null +++ b/contrib/git-changes-action/detector/module/detector.go @@ -0,0 +1,59 @@ +package moduledetector + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" + "golang.org/x/mod/modfile" + "os" + "path" +) + +// DetectChangedModules is the change detector client. +// Will flag modules as changed if any module in their dependency tree has been modified. +// nolint: cyclop +func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modules map[string]bool, err error) { + modules = make(map[string]bool) + + goWorkPath := path.Join(repoPath, "go.work") + + if !common.FileExist(goWorkPath) { + return nil, fmt.Errorf("go.work file not found in %s", repoPath) + } + + //nolint: gosec + workFile, err := os.ReadFile(goWorkPath) + if err != nil { + return nil, fmt.Errorf("failed to read go.work file: %w", err) + } + + parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse go.work file: %w", err) + } + + depGraph, err := getDependencyGraph(repoPath) + if err != nil { + return nil, fmt.Errorf("could not get dep graph: %w", err) + } + + for _, module := range parsedWorkFile.Use { + changed := false + if ct.HasPath(module.Path) { + changed = true + } + + if includeDeps { + deps := depGraph[module.Path] + for _, dep := range deps { + if ct.HasPath(dep) { + changed = true + } + } + } + + modules[module.Path] = changed + } + + return modules, nil +} diff --git a/contrib/git-changes-action/detector/detector_test.go b/contrib/git-changes-action/detector/module/detector_test.go similarity index 76% rename from contrib/git-changes-action/detector/detector_test.go rename to contrib/git-changes-action/detector/module/detector_test.go index e82c5e3d76..699f750ec0 100644 --- a/contrib/git-changes-action/detector/detector_test.go +++ b/contrib/git-changes-action/detector/module/detector_test.go @@ -1,18 +1,21 @@ -package detector_test +package moduledetector_test import ( "encoding/hex" + "fmt" "github.com/ethereum/go-ethereum/crypto" . "github.com/stretchr/testify/assert" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/actionscore" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/git" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/gitmock" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/module" "os" "path/filepath" ) func (d *DetectorSuite) TestChangedModules() { testRepo, err := gitmock.NewTestRepo(d.T(), d.sourceRepo.repo, d.sourceRepo.dir) + fmt.Println("TEST REPO: ", testRepo) Nil(d.T(), err, "should not return an error") _, err = os.Create(filepath.Join(d.sourceRepo.dir, "lib", "newfile.go")) @@ -20,13 +23,14 @@ func (d *DetectorSuite) TestChangedModules() { testRepo.Commit() - ct, err := detector.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") + ct, err := git.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") + fmt.Println(ct) Nil(d.T(), err, "should not return an error") - withDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, true, "modules") + withDeps, err := moduledetector.DetectChangedModules(d.sourceRepo.dir, ct, true) Nil(d.T(), err, "should not return an error") - withoutDeps, err := detector.DetectChangedModules(d.sourceRepo.dir, ct, false, "modules") + withoutDeps, err := moduledetector.DetectChangedModules(d.sourceRepo.dir, ct, false) Nil(d.T(), err, "should not return an error") False(d.T(), withoutDeps["./cmd/app1"]) @@ -41,7 +45,7 @@ func (d *DetectorSuite) TestChangedModules() { } func (d *DetectorSuite) TestGetDependencyDag() { - deps, _, err := detector.GetDependencyDag(d.sourceRepo.dir) + deps, err := moduledetector.GetDependencyDag(d.sourceRepo.dir) Nil(d.T(), err, "should not return an error") Equal(d.T(), deps["./cmd/app1"], []string{"./lib"}) @@ -57,14 +61,14 @@ func (d *DetectorSuite) TestGetHead() { hash := testRepo.Commit() - head, err := detector.GetHead(d.sourceRepo.repo, &actionscore.Context{ + head, err := git.GetHead(d.sourceRepo.repo, &actionscore.Context{ Ref: hash, }, "") Nil(d.T(), err, "should not return an error") Equal(d.T(), head, hash) randHash := hex.EncodeToString(crypto.Keccak256([]byte("random hash"))) - head, err = detector.GetHead(d.sourceRepo.repo, &actionscore.Context{ + head, err = git.GetHead(d.sourceRepo.repo, &actionscore.Context{ Ref: hash, }, randHash) Nil(d.T(), err, "should not return an error") @@ -77,7 +81,7 @@ func (d *DetectorSuite) TestChangeTree() { addedFiles := testRepo.AddRandomFiles(5) - changeTree, err := detector.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") + changeTree, err := git.GetChangeTree(d.GetTestContext(), d.sourceRepo.dir, "", "", "main") Nil(d.T(), err, "should not empty change tree") for _, file := range addedFiles { diff --git a/contrib/git-changes-action/detector/module/doc.go b/contrib/git-changes-action/detector/module/doc.go new file mode 100644 index 0000000000..cbf76017ad --- /dev/null +++ b/contrib/git-changes-action/detector/module/doc.go @@ -0,0 +1,3 @@ +// Package moduledetector implements a detector for detecting changes in a git repo. +// Dependencies are transitive, at the module level. Modules depend on modules. +package moduledetector diff --git a/contrib/git-changes-action/detector/module/export_test.go b/contrib/git-changes-action/detector/module/export_test.go new file mode 100644 index 0000000000..bf1f265434 --- /dev/null +++ b/contrib/git-changes-action/detector/module/export_test.go @@ -0,0 +1,5 @@ +package moduledetector + +func GetDependencyDag(repoPath string) (map[string][]string, error) { + return getDependencyGraph(repoPath) +} diff --git a/contrib/git-changes-action/detector/suite_test.go b/contrib/git-changes-action/detector/module/suite_test.go similarity index 99% rename from contrib/git-changes-action/detector/suite_test.go rename to contrib/git-changes-action/detector/module/suite_test.go index 494a267978..c8235ca99e 100644 --- a/contrib/git-changes-action/detector/suite_test.go +++ b/contrib/git-changes-action/detector/module/suite_test.go @@ -1,4 +1,4 @@ -package detector_test +package moduledetector_test import ( "github.com/Flaque/filet" diff --git a/contrib/git-changes-action/detector/package/depgraph.go b/contrib/git-changes-action/detector/package/depgraph.go new file mode 100644 index 0000000000..d6c3664359 --- /dev/null +++ b/contrib/git-changes-action/detector/package/depgraph.go @@ -0,0 +1,202 @@ +package packagedetector + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/kendru/darwin/go/depgraph" + "github.com/vishalkuo/bimap" + "go/parser" + "go/token" + "golang.org/x/mod/modfile" + "os" + "path" + "path/filepath" + "strings" +) + +// nolint: cyclop +func getPackageDependencyGrap(repoPath string) (moduleDeps map[string][]string, packagesPerModule map[string][]string, err error) { + moduleDeps = make(map[string][]string) + // parse the go.work file + goWorkPath := path.Join(repoPath, "go.work") + + if !common.FileExist(goWorkPath) { + return nil, nil, fmt.Errorf("go.work file not found in %s", repoPath) + } + + //nolint: gosec + workFile, err := os.ReadFile(goWorkPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read go.work file: %w", err) + } + + parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse go.work file: %w", err) + } + + // map of package->dependencies + var dependencies map[string]map[string]struct{} + + // iterate through each module in the go.work file + // create a list of dependencies for each package + // and generate a list of packages per module + // nolint: gocognit + dependencies, packagesPerModule, err = makePackageDepMaps(repoPath, parsedWorkFile.Use) + if err != nil { + return nil, nil, fmt.Errorf("failed to create dependency maps: %w", err) + } + + depGraph := depgraph.New() + + for _, module := range parsedWorkFile.Use { + for _, relativePackageName := range packagesPerModule[module.Path] { + for relativePackageDependencyName := range dependencies[relativePackageName] { + err = depGraph.DependOn(relativePackageName, relativePackageDependencyName) + // Circular dependencies are fine as long as both packages are in the same module + if err != nil && !(strings.Contains(relativePackageDependencyName, module.Path) && strings.Contains(relativePackageName, module.Path)) { + return nil, nil, fmt.Errorf("failed to add dependency %s -> %s: %w", relativePackageName, relativePackageDependencyName, err) + } + } + } + } + + for _, module := range parsedWorkFile.Use { + for _, relativePackageName := range packagesPerModule[module.Path] { + for dep := range depGraph.Dependencies(relativePackageName) { + moduleDeps[relativePackageName] = append(moduleDeps[relativePackageName], dep) + } + } + } + return moduleDeps, packagesPerModule, nil +} + +func extractGoFileNames(pwd string, currentPackage string, goFiles map[string][]string) (err error) { + searchNext := make(map[string]string) + _, packageDir := path.Split(currentPackage) + searchNext[pwd] = packageDir + + for len(searchNext) > 0 { + discovered := make(map[string]string) + for path, dirName := range searchNext { + err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() && !(path == filePath) { + discovered[filePath] = info.Name() + return filepath.SkipDir + } else if strings.HasSuffix(info.Name(), ".go") { + goFiles["/"+dirName] = append(goFiles["/"+dirName], filePath) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to walk path: %w", err) + } + } + searchNext = discovered + } + return nil +} + +// nolint: gocognit, cyclop +func makePackageDepMaps(repoPath string, uses []*modfile.Use) (dependencies map[string]map[string]struct{}, packagesPerModule map[string][]string, err error) { + // map of packages -> dependencies + dependencies = make(map[string]map[string]struct{}) + + // bidirectional map of package->package name + // Maps relative to public names, used to filer out all external libraries/packages. + dependencyNames := bimap.NewBiMap[string, string]() + + // map of module->packages + packagesPerModule = make(map[string][]string) + + // map module->package->goFiles + // Maps each module to all packages and each package to its go files. + extractedGoFileNames := make(map[string]map[string][]string) + + pwd, err := os.Getwd() + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to read current directory: %w", err) + } + // iterate through each module in the go.work file + // 1. Extract all go files filepaths for each package. + // 2. Create a map where key is module, value is an array with all packages in the module + // 3. Map public name to relative name for each package (used to filter external library/package imports) + for _, module := range uses { + // nolint: gosec + modContents, err := os.ReadFile(filepath.Join(repoPath, module.Path, "go.mod")) + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to read module file %s: %w", module.Path, err) + } + + parsedModFile, err := modfile.Parse(module.Path, modContents, nil) + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to parse module file %s: %w", module.Path, err) + } + + extractedGoFileNames[module.Path] = make(map[string][]string) + err = extractGoFileNames(pwd+module.Path[1:], module.Path[1:], extractedGoFileNames[module.Path]) + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to extract go files for module %s: %w", module.Path, err) + } + + for packageName := range extractedGoFileNames[module.Path] { + var relativePackageName string + if strings.HasSuffix(module.Path, packageName) { + relativePackageName = module.Path + } else { + relativePackageName = module.Path + packageName + } + + var publicPackageName string + if strings.HasSuffix(parsedModFile.Module.Mod.Path, packageName) { + publicPackageName = parsedModFile.Module.Mod.Path + } else { + publicPackageName = parsedModFile.Module.Mod.Path + packageName + } + + packagesPerModule[module.Path] = append(packagesPerModule[module.Path], relativePackageName) + dependencyNames.Insert(relativePackageName, publicPackageName) + } + } + + // iterate through each module in the go.work file + // For every package in the module + // using the filepaths extracted on the previous loop, parse files and extract imports. + // Ignore any external library/package imports + for _, module := range uses { + for packageInModule, files := range extractedGoFileNames[module.Path] { + var relativePackageName string + if strings.HasSuffix(module.Path, packageInModule) { + relativePackageName = module.Path + } else { + relativePackageName = module.Path + packageInModule + } + + dependencies[relativePackageName] = make(map[string]struct{}) + for _, file := range files { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + + if err != nil { + return dependencies, packagesPerModule, fmt.Errorf("failed to parse go file %s in package %s: %w", file, relativePackageName, err) + } + + for _, s := range f.Imports { + // s.Path.Value contains double quotation marks that must be removed before indexing dependencyNames + renamedDep, hasDep := dependencyNames.GetInverse(s.Path.Value[1 : len(s.Path.Value)-1]) + + if hasDep && (relativePackageName != renamedDep) { + dependencies[relativePackageName][renamedDep] = struct{}{} + } + } + } + } + } + return dependencies, packagesPerModule, nil +} diff --git a/contrib/git-changes-action/detector/package/detector.go b/contrib/git-changes-action/detector/package/detector.go new file mode 100644 index 0000000000..63e3e41ada --- /dev/null +++ b/contrib/git-changes-action/detector/package/detector.go @@ -0,0 +1,76 @@ +package packagedetector + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" + "golang.org/x/mod/modfile" + "os" + "path" +) + +// DetectChangedModules is the change detector client. +// Modules will be flagged as changed if any of the packaged in their dependency tree has been modified. +// nolint: cyclop +func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modules map[string]bool, err error) { + modules = make(map[string]bool) + + goWorkPath := path.Join(repoPath, "go.work") + + if !common.FileExist(goWorkPath) { + return nil, fmt.Errorf("go.work file not found in %s", repoPath) + } + + //nolint: gosec + workFile, err := os.ReadFile(goWorkPath) + if err != nil { + return nil, fmt.Errorf("failed to read go.work file: %w", err) + } + + parsedWorkFile, err := modfile.ParseWork(goWorkPath, workFile, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse go.work file: %w", err) + } + + depGraph, packagesPerModule, err := getPackageDependencyGrap(repoPath) + if err != nil { + return nil, fmt.Errorf("could not get dep graph: %w", err) + } + + for _, module := range parsedWorkFile.Use { + changed := false + + if ct.HasPath(module.Path) { + changed = true + } + + if includeDeps && !changed { + for _, packageName := range packagesPerModule[module.Path] { + if isPackageChanged(packageName, ct, depGraph) { + changed = true + // If a package is flagged as changed + // its not necessary to analyze the remaining packages, + // module can be flagged as changed + break + } + } + } + + modules[module.Path] = changed + } + + return modules, nil +} + +func isPackageChanged(packageName string, ct tree.Tree, depGraph map[string][]string) bool { + if ct.HasPath(packageName) { + return true + } + + for _, dep := range depGraph[packageName] { + if ct.HasPath(dep) { + return true + } + } + return false +} diff --git a/contrib/git-changes-action/detector/package/doc.go b/contrib/git-changes-action/detector/package/doc.go new file mode 100644 index 0000000000..085a40b791 --- /dev/null +++ b/contrib/git-changes-action/detector/package/doc.go @@ -0,0 +1,3 @@ +// Package packagedetector implements a detector for detecting changes in a git repo. +// Dependencies are transitive, at the package level. Modules depend on packages, whose dep graph is also at package level. Module->package->package +package packagedetector diff --git a/contrib/git-changes-action/main.go b/contrib/git-changes-action/main.go index 5d02e1ab58..ad63817c64 100644 --- a/contrib/git-changes-action/main.go +++ b/contrib/git-changes-action/main.go @@ -5,14 +5,16 @@ import ( "context" "encoding/json" "fmt" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" "os" "sort" "strings" "time" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/package" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" + "github.com/sethvargo/go-githubactions" - "github.com/synapsecns/sanguine/contrib/git-changes-action/detector" + "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/git" ) const defaultTimeout = "1m" @@ -36,17 +38,17 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - ct, err := detector.GetChangeTree(ctx, workingDirectory, ref, token, base) + ct, err := git.GetChangeTree(ctx, workingDirectory, ref, token, base) if err != nil { panic(err) } - noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false, "packages") + noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false) if err != nil { panic(err) } - depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true, "packages") + depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true) if err != nil { panic(err) } @@ -61,8 +63,8 @@ func main() { // outputModuleChanges outputs the changed modules. // this wraps detector.DetectChangedModules and handles the output formatting to be parsable by github actions. // the final output is a json array of strings. -func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool, typeOfDependency string) (changedJSON string, unchangedJson string, err error) { - modules, err := detector.DetectChangedModules(workingDirectory, ct, includeDeps, typeOfDependency) +func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool) (changedJSON string, unchangedJson string, err error) { + modules, err := packagedetector.DetectChangedModules(workingDirectory, ct, includeDeps) if err != nil { return changedJSON, unchangedJson, fmt.Errorf("failed to detect changed modules w/ include deps set to %v: %w", includeDeps, err) } From 1ae1307295c011f54bd859a00b2977830ff2ac16 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:33:32 -0600 Subject: [PATCH 28/39] fix: typo --- contrib/git-changes-action/detector/package/depgraph.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/git-changes-action/detector/package/depgraph.go b/contrib/git-changes-action/detector/package/depgraph.go index d6c3664359..8636d964dd 100644 --- a/contrib/git-changes-action/detector/package/depgraph.go +++ b/contrib/git-changes-action/detector/package/depgraph.go @@ -39,9 +39,9 @@ func getPackageDependencyGrap(repoPath string) (moduleDeps map[string][]string, var dependencies map[string]map[string]struct{} // iterate through each module in the go.work file - // create a list of dependencies for each package - // and generate a list of packages per module - // nolint: gocognit + // create a list of dependencies for each package + // and generate a list of packages per module + // nolint: gocognit dependencies, packagesPerModule, err = makePackageDepMaps(repoPath, parsedWorkFile.Use) if err != nil { return nil, nil, fmt.Errorf("failed to create dependency maps: %w", err) From 5d3060d79a4271a6d2c95e170cd4604904757f31 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:34:26 -0600 Subject: [PATCH 29/39] fix: typo in function name --- contrib/git-changes-action/detector/package/depgraph.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/git-changes-action/detector/package/depgraph.go b/contrib/git-changes-action/detector/package/depgraph.go index 8636d964dd..7c10a42cb8 100644 --- a/contrib/git-changes-action/detector/package/depgraph.go +++ b/contrib/git-changes-action/detector/package/depgraph.go @@ -15,7 +15,7 @@ import ( ) // nolint: cyclop -func getPackageDependencyGrap(repoPath string) (moduleDeps map[string][]string, packagesPerModule map[string][]string, err error) { +func getPackageDependencyGraph(repoPath string) (moduleDeps map[string][]string, packagesPerModule map[string][]string, err error) { moduleDeps = make(map[string][]string) // parse the go.work file goWorkPath := path.Join(repoPath, "go.work") From 8d2fd3ced49dbd20147cf63706668acca3b71e68 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:43:51 -0600 Subject: [PATCH 30/39] fix: lint issue moving variable on range scope to local scope, can be safely used in function --- contrib/git-changes-action/detector/package/depgraph.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/git-changes-action/detector/package/depgraph.go b/contrib/git-changes-action/detector/package/depgraph.go index 7c10a42cb8..f3a9d68cce 100644 --- a/contrib/git-changes-action/detector/package/depgraph.go +++ b/contrib/git-changes-action/detector/package/depgraph.go @@ -79,6 +79,9 @@ func extractGoFileNames(pwd string, currentPackage string, goFiles map[string][] for len(searchNext) > 0 { discovered := make(map[string]string) for path, dirName := range searchNext { + // see https://stackoverflow.com/a/68559720 + dirName := dirName + path := path err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { if err != nil { return err From 07d04a70d3b0e062c443b5311b4900846a203610 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:47:04 -0600 Subject: [PATCH 31/39] fix: lint issues --- contrib/git-changes-action/detector/package/depgraph.go | 6 +++--- contrib/git-changes-action/detector/package/detector.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/git-changes-action/detector/package/depgraph.go b/contrib/git-changes-action/detector/package/depgraph.go index f3a9d68cce..a6898e3d0a 100644 --- a/contrib/git-changes-action/detector/package/depgraph.go +++ b/contrib/git-changes-action/detector/package/depgraph.go @@ -79,9 +79,9 @@ func extractGoFileNames(pwd string, currentPackage string, goFiles map[string][] for len(searchNext) > 0 { discovered := make(map[string]string) for path, dirName := range searchNext { - // see https://stackoverflow.com/a/68559720 - dirName := dirName - path := path + // see https://stackoverflow.com/a/68559720 + dirName := dirName + path := path err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { if err != nil { return err diff --git a/contrib/git-changes-action/detector/package/detector.go b/contrib/git-changes-action/detector/package/detector.go index 63e3e41ada..0f8fd52cb2 100644 --- a/contrib/git-changes-action/detector/package/detector.go +++ b/contrib/git-changes-action/detector/package/detector.go @@ -32,7 +32,7 @@ func DetectChangedModules(repoPath string, ct tree.Tree, includeDeps bool) (modu return nil, fmt.Errorf("failed to parse go.work file: %w", err) } - depGraph, packagesPerModule, err := getPackageDependencyGrap(repoPath) + depGraph, packagesPerModule, err := getPackageDependencyGraph(repoPath) if err != nil { return nil, fmt.Errorf("could not get dep graph: %w", err) } From 586ab76d6e86a9809ac6e6251ed6f8344c6a97c0 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sat, 30 Mar 2024 13:23:37 -0600 Subject: [PATCH 32/39] Update eventtype_string.go --- contrib/git-changes-action/detector/git/eventtype_string.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/git-changes-action/detector/git/eventtype_string.go b/contrib/git-changes-action/detector/git/eventtype_string.go index 975faecc64..3cb7df1226 100644 --- a/contrib/git-changes-action/detector/git/eventtype_string.go +++ b/contrib/git-changes-action/detector/git/eventtype_string.go @@ -1,5 +1,7 @@ // Code generated by "stringer -type=EventType -linecomment"; DO NOT EDIT. -package git + +package git + import "strconv" func _() { From faf1fdae16eeb5a63471612c6227b036d4dcc494 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Sun, 31 Mar 2024 19:35:49 -0700 Subject: [PATCH 33/39] chore: Update READ ME WIP, need to clean graph --- contrib/git-changes-action/README.md | 95 ++++++++++++++++------------ 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/contrib/git-changes-action/README.md b/contrib/git-changes-action/README.md index 7225aa94b3..cb347106f6 100644 --- a/contrib/git-changes-action/README.md +++ b/contrib/git-changes-action/README.md @@ -1,6 +1,11 @@ -# Git Changes Action +Git Changes Action -This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request, along with any dependent modules. This can be useful for automating build, test, or deployment workflows that involve Go projects. +This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request. This can be useful for automating build, test, or deployment workflows that involve Go projects. + +The library can take two approaches to dependency resolution: + +1. Module to module. This means that if any package in ModuleA that is part of the dependency tree of ModuleB will be flagged as changed, regardless of whether the change at hand affects a package that moduleB directly imports or not. +2. Module to package. This means that if packageA in ModuleA is changed, only modules directly or indirectly depending on packageA will be flagged as changed. [![Go Reference](https://pkg.go.dev/badge/github.com/synapsecns/sanguine/contrib/git-changes-action.svg)](https://pkg.go.dev/github.com/synapsecns/sanguine/contrib/git-changes-action) [![Go Report Card](https://goreportcard.com/badge/github.com/synapsecns/sanguine/contrib/git-changes-action)](https://goreportcard.com/report/github.com/synapsecns/sanguine/contrib/git-changes-action) @@ -21,7 +26,7 @@ This GitHub Action exports a variable that contains the list of Go modules chang submodules: 'recursive' ``` -1. Use the synapsecns/sanguine/git-changes-action Docker image to run the git-changes script, which exports a variable that contains the list of changed Go modules. +2. Use the synapsecns/sanguine/git-changes-action Docker image to run the git-changes script, which exports a variable that contains the list of changed Go modules. ```yaml - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest @@ -31,10 +36,14 @@ This GitHub Action exports a variable that contains the list of Go modules chang timeout: "1m" # optional, defaults to 1m ``` -You can customize the behavior of the git-changes script by using the following inputs: +3. You can customize the behavior of the git-changes script by using the following inputs: - `github_token`: The token to use for authentication with the GitHub API. This is required to fetch information about the current pull request. - `timeout`: The maximum time to wait for the GitHub API to respond. Defaults to 1 minute. + - `levelOfDependencyResolution`: This parameter will determine if its dependency resolution is be based from + 1. "modules" `module to module` + 2. "packages" `module to packages` + The output of the git-changes script is a comma-separated list of Go module paths. You can access this list using the `filter_go` output variable, like so: @@ -45,10 +54,46 @@ The output of the git-changes script is a comma-separated list of Go module path - run: echo "Unchanged modules (including dependencies): ${{ steps.filter_go.outputs.unchanged_modules_deps }}" ``` -## Example +Note that the result will be heavily influenced by the type of `levelOfDependencyResolution` chosen by the user. + +## Example 1 + +Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules on a modules `levelOfDepencencyResolution`: + +```yaml +name: Test Go Modules + +on: + pull_request: + types: [opened, synchronize] + +jobs: + test_go_modules: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: 'recursive' + + - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest + id: filter_go + with: + github_token: ${{ secrets.github_token }} + timeout: "1m" + typeOfDependencyResolution: "modules" + + + - name: Run tests + run: go test -v ${{ steps.filter_go.outputs.changed_modules }} +``` + +This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. Modules will be flagged as changed if their module dependencies have been changed. -Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules: +## Example 2 + +Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules on a packages `levelOfDepencencyResolution`: ```yaml name: Test Go Modules @@ -71,12 +116,15 @@ jobs: with: github_token: ${{ secrets.github_token }} timeout: "1m" + typeOfDependencyResolution: "packages" + - name: Run tests run: go test -v ${{ steps.filter_go.outputs.changed_modules }} ``` -This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. +This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. Modules will be flagged as changed if any of the package dependencies in their dependency tree have been changed. + ## How It Works @@ -93,36 +141,3 @@ graph TB E --> G F --> G ``` - -Each module in the `go.work` is visited. If any changes were detected by the previous step, the module is added to the list of changed modules. If `include_dependencies` is on and the module has a dependency that is also in the `go.work`, the dependency is added to the list of changed modules as well. This process is repeated until all modules in the `go.work` have been visited. - -```mermaid -sequenceDiagram - participant GW as go.work - participant M as Module - participant CML as Changed_Module_List - participant UML as Unchanged_Module_List - participant D as Dependency - - GW->>M: Visit Module - Note over M: Check for changes - M-->>GW: Changes Detected? - alt Changes Detected - GW->>CML: Add Module to Changed_Module_List - M->>D: Has Dependency in go.work? - alt Has Dependency - GW->>CML: Add Dependency to Changed_Module_List - else No Dependency - M-->>GW: No Dependency to Add - end - else No Changes Detected - GW->>UML: Add Module to Unchanged_Module_List - M->>D: Has Dependency in go.work? - alt Has Dependency - GW->>UML: Add Dependency to Unchanged_Module_List - else No Dependency - M-->>GW: No Dependency to Add - end - end - GW->>GW: Continue Until All Modules Visited -``` From a33e94c878f5446f2ccbc63f9993777e38837c02 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:21:29 -0600 Subject: [PATCH 34/39] Update README.md --- contrib/git-changes-action/README.md | 112 +++++++++++++++++---------- 1 file changed, 70 insertions(+), 42 deletions(-) diff --git a/contrib/git-changes-action/README.md b/contrib/git-changes-action/README.md index cb347106f6..f52df1ff57 100644 --- a/contrib/git-changes-action/README.md +++ b/contrib/git-changes-action/README.md @@ -1,11 +1,6 @@ -Git Changes Action +# Git Changes Action -This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request. This can be useful for automating build, test, or deployment workflows that involve Go projects. - -The library can take two approaches to dependency resolution: - -1. Module to module. This means that if any package in ModuleA that is part of the dependency tree of ModuleB will be flagged as changed, regardless of whether the change at hand affects a package that moduleB directly imports or not. -2. Module to package. This means that if packageA in ModuleA is changed, only modules directly or indirectly depending on packageA will be flagged as changed. +This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request, and if desierd, along with any dependent modules. This can be useful for automating build, test, or deployment workflows that involve Go projects. [![Go Reference](https://pkg.go.dev/badge/github.com/synapsecns/sanguine/contrib/git-changes-action.svg)](https://pkg.go.dev/github.com/synapsecns/sanguine/contrib/git-changes-action) [![Go Report Card](https://goreportcard.com/badge/github.com/synapsecns/sanguine/contrib/git-changes-action)](https://goreportcard.com/report/github.com/synapsecns/sanguine/contrib/git-changes-action) @@ -16,17 +11,16 @@ The library can take two approaches to dependency resolution: Check out the current pull request using the actions/checkout action. It's recommended to set fetch-depth to 0 and submodules to recursive to ensure that all necessary dependencies are fetched. - - ```yaml steps: - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: 'recursive' + packageOfLevelResolution: "modules" ``` -2. Use the synapsecns/sanguine/git-changes-action Docker image to run the git-changes script, which exports a variable that contains the list of changed Go modules. +1. Use the synapsecns/sanguine/git-changes-action Docker image to run the git-changes script, which exports a variable that contains the list of changed Go modules. ```yaml - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest @@ -34,16 +28,14 @@ The library can take two approaches to dependency resolution: with: github_token: ${{ secrets.github_token }} timeout: "1m" # optional, defaults to 1m + packageLevelResolution: "modules" "packages" ``` -3. You can customize the behavior of the git-changes script by using the following inputs: +You can customize the behavior of the git-changes script by using the following inputs: - `github_token`: The token to use for authentication with the GitHub API. This is required to fetch information about the current pull request. - `timeout`: The maximum time to wait for the GitHub API to respond. Defaults to 1 minute. - - `levelOfDependencyResolution`: This parameter will determine if its dependency resolution is be based from - 1. "modules" `module to module` - 2. "packages" `module to packages` - + - 'packageLevelResolution: "modules" or "packages" The output of the git-changes script is a comma-separated list of Go module paths. You can access this list using the `filter_go` output variable, like so: @@ -54,11 +46,26 @@ The output of the git-changes script is a comma-separated list of Go module path - run: echo "Unchanged modules (including dependencies): ${{ steps.filter_go.outputs.unchanged_modules_deps }}" ``` -Note that the result will be heavily influenced by the type of `levelOfDependencyResolution` chosen by the user. +## How It Works + +First a change tree is calculated between the ref and the base (as passed in by the user or inferred by the event type) based on the flow described below and [here](https://github.com/dorny/paths-filter/blob/4067d885736b84de7c414f582ac45897079b0a78/README.md#supported-workflows). This is a file tree that contains all the files that have changed between two refs. + +```mermaid +graph TB + A(Start) --> B{Check triggering workflow} + B -- pull request --> C[Calculate a list diff with the GitHub API] + B -- push --> D{Check base} + D -- Base same as head --> E[Changes detected against most recent commit] + D -- Base is a commit sha --> F[Changes detected against the sha] + C --> G(Generate Chagned File List) + E --> G + F --> G +``` + -## Example 1 +## Module Level Dependency Resolution -Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules on a modules `levelOfDepencencyResolution`: +Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules: ```yaml name: Test Go Modules @@ -79,21 +86,59 @@ jobs: - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest id: filter_go with: + packageLevelResolution: "modules" github_token: ${{ secrets.github_token }} timeout: "1m" - typeOfDependencyResolution: "modules" - - name: Run tests run: go test -v ${{ steps.filter_go.outputs.changed_modules }} ``` -This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. Modules will be flagged as changed if their module dependencies have been changed. +This workflow will run tests for all changed Go modules and their module dependencies whenever a pull request is opened or synchronized. -## Example 2 +## Module Level Resolution -Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules on a packages `levelOfDepencencyResolution`: +Each module in the `go.work` is visited. If any changes were detected, the module itself it is added to the list of changed modules. If `include_dependencies` is on, and their direct or indirect module dependencies are modified the module is flagged as changed. This process is repeated until all modules in the `go.work` have been visited. + +```mermaid +sequenceDiagram + participant GW as go.work + participant M as Module + participant P as Package + participant CML as Changed_Module_List + participant UML as Unchanged_Module_List + participant D as Dependency + + GW->>M: Visit Module + M->>P:Extract all packages for each Module + P->>M: Returns [module]: [package1,package2] +M->>M: Changes detected? +M->>P: Loop over packages + P-->>GW: Changes Detected? + alt Changes Detected + GW->>CML: Add Module to Changed_Module_List + M->>D: Has Dependency in go.work? + alt Has Dependency + GW->>CML: Add Dependency to Changed_Module_List + else No Dependency + M-->>GW: No Dependency to Add + end + else No Changes Detected + GW->>UML: Add Module to Unchanged_Module_List + M->>D: Has Dependency in go.work? + alt Has Dependency + GW->>UML: Add Dependency to Unchanged_Module_List + else No Dependency + M-->>GW: No Dependency to Add + end + end + GW->>GW: Continue Until All Modules Visited +``` + +## Package Level Resolution + +Each module in the `go.work` is visited. And the module has been changed, the module itself is added to the list of changed modules. If `include_dependencies` is on, if their direct or indirect package dependencies are modified the module is flagged as changed. This process is repeated until all modules dependencies been visited. ```yaml name: Test Go Modules @@ -114,30 +159,13 @@ jobs: - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest id: filter_go with: + packageLevelResolution: "modules" github_token: ${{ secrets.github_token }} timeout: "1m" - typeOfDependencyResolution: "packages" - - name: Run tests run: go test -v ${{ steps.filter_go.outputs.changed_modules }} ``` -This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. Modules will be flagged as changed if any of the package dependencies in their dependency tree have been changed. - - -## How It Works - -First a change tree is calculated between the ref and the base (as passed in by the user or inferred by the event type) based on the flow described below and [here](https://github.com/dorny/paths-filter/blob/4067d885736b84de7c414f582ac45897079b0a78/README.md#supported-workflows). This is a file tree that contains all the files that have changed between two refs. +This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. -```mermaid -graph TB - A(Start) --> B{Check triggering workflow} - B -- pull request --> C[Calculate a list diff with the GitHub API] - B -- push --> D{Check base} - D -- Base same as head --> E[Changes detected against most recent commit] - D -- Base is a commit sha --> F[Changes detected against the sha] - C --> G(Generate Chagned File List) - E --> G - F --> G -``` From 12dc6f5f7255bb5a7f6f1cff03dc66c99a5e23b7 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 3 Apr 2024 07:34:27 -0600 Subject: [PATCH 35/39] Update README --- contrib/git-changes-action/README.md | 115 ++++++++++----------------- 1 file changed, 41 insertions(+), 74 deletions(-) diff --git a/contrib/git-changes-action/README.md b/contrib/git-changes-action/README.md index f52df1ff57..100db6a378 100644 --- a/contrib/git-changes-action/README.md +++ b/contrib/git-changes-action/README.md @@ -1,6 +1,6 @@ # Git Changes Action -This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request, and if desierd, along with any dependent modules. This can be useful for automating build, test, or deployment workflows that involve Go projects. +This GitHub Action exports a variable that contains the list of Go modules changed in the current pull request, and if desired, along with any dependent modules. This can be useful for automating build, test, or deployment workflows that involve Go projects. [![Go Reference](https://pkg.go.dev/badge/github.com/synapsecns/sanguine/contrib/git-changes-action.svg)](https://pkg.go.dev/github.com/synapsecns/sanguine/contrib/git-changes-action) [![Go Report Card](https://goreportcard.com/badge/github.com/synapsecns/sanguine/contrib/git-changes-action)](https://goreportcard.com/report/github.com/synapsecns/sanguine/contrib/git-changes-action) @@ -28,22 +28,23 @@ This GitHub Action exports a variable that contains the list of Go modules chang with: github_token: ${{ secrets.github_token }} timeout: "1m" # optional, defaults to 1m - packageLevelResolution: "modules" "packages" + dependencyLevelResolution: "modules" or "packages" ``` You can customize the behavior of the git-changes script by using the following inputs: - `github_token`: The token to use for authentication with the GitHub API. This is required to fetch information about the current pull request. - `timeout`: The maximum time to wait for the GitHub API to respond. Defaults to 1 minute. - - 'packageLevelResolution: "modules" or "packages" + - `dependencyLevelResolution`: "modules" or "packages" The output of the git-changes script is a comma-separated list of Go module paths. You can access this list using the `filter_go` output variable, like so: ```yaml - - run: echo "Changed modules: ${{ steps.filter_go.outputs.changed_modules }}" - - run: echo "Unchanged modules: ${{ steps.filter_go.outputs.unchanged_modules }}" - - run: echo "Changed modules (including dependencies): ${{ steps.filter_go.outputs.changed_modules_deps }}" - - run: echo "Unchanged modules (including dependencies): ${{ steps.filter_go.outputs.unchanged_modules_deps }}" + - run: echo 'Changed modules:' ${{ steps.filter_go.outputs.changed_modules }} + - run: echo 'Unchanged modules:' ${{ steps.filter_go.outputs.unchanged_modules }} + - run: echo 'Changed modules (including dependencies):' ${{ steps.filter_go.outputs.changed_modules_deps }} + - run: echo 'Unchanged modules (including dependencies):' ${{ steps.filter_go.outputs.unchanged_modules_deps }} + ``` ## How It Works @@ -62,44 +63,44 @@ graph TB F --> G ``` +## Module Level Resolution -## Module Level Dependency Resolution - -Here's an example workflow that uses the `git-changes` action to run tests for changed Go modules: - -```yaml -name: Test Go Modules - -on: - pull_request: - types: [opened, synchronize] +Each module in the `go.work` is visited. If any changes are detected, the module itself is added to the list of changed modules. By setting `include_dependencies` to `modules`, changes to their direct or indirect module dependencies will flag the module as changed. This process is repeated until all modules in the `go.work` have been visited. -jobs: - test_go_modules: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'recursive' - - - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest - id: filter_go - with: - packageLevelResolution: "modules" - github_token: ${{ secrets.github_token }} - timeout: "1m" +```mermaid +sequenceDiagram + participant GW as go.work + participant M as Module + participant CML as Changed_Module_List + participant UML as Unchanged_Module_List + participant D as Dependency - - name: Run tests - run: go test -v ${{ steps.filter_go.outputs.changed_modules }} + GW->>M: Visit Module + Note over M: Check for changes + M-->>GW: Changes Detected? + alt Changes Detected + GW->>CML: Add Module to Changed_Module_List + M->>D: Has Dependency in go.work? + alt Has Dependency + GW->>CML: Add Dependency to Changed_Module_List + else No Dependency + M-->>GW: No Dependency to Add + end + else No Changes Detected + GW->>UML: Add Module to Unchanged_Module_List + M->>D: Has Dependency in go.work? + alt Has Dependency + GW->>UML: Add Dependency to Unchanged_Module_List + else No Dependency + M-->>GW: No Dependency to Add + end + end + GW->>GW: Continue Until All Modules Visited ``` -This workflow will run tests for all changed Go modules and their module dependencies whenever a pull request is opened or synchronized. - - -## Module Level Resolution +## Package Level Resolution -Each module in the `go.work` is visited. If any changes were detected, the module itself it is added to the list of changed modules. If `include_dependencies` is on, and their direct or indirect module dependencies are modified the module is flagged as changed. This process is repeated until all modules in the `go.work` have been visited. +Each module in the `go.work` is visited. If any changes are detected, the module itself is added to the list of changed modules. By setting `include_dependencies` to `packages`, changes to its direct or indirect package dependencies will flag the module as changed. This process is repeated until all package dependencies for the module been visited. ```mermaid sequenceDiagram @@ -118,7 +119,7 @@ M->>P: Loop over packages P-->>GW: Changes Detected? alt Changes Detected GW->>CML: Add Module to Changed_Module_List - M->>D: Has Dependency in go.work? + M->>D: Has Package Dependency? alt Has Dependency GW->>CML: Add Dependency to Changed_Module_List else No Dependency @@ -135,37 +136,3 @@ M->>P: Loop over packages end GW->>GW: Continue Until All Modules Visited ``` - -## Package Level Resolution - -Each module in the `go.work` is visited. And the module has been changed, the module itself is added to the list of changed modules. If `include_dependencies` is on, if their direct or indirect package dependencies are modified the module is flagged as changed. This process is repeated until all modules dependencies been visited. - -```yaml -name: Test Go Modules - -on: - pull_request: - types: [opened, synchronize] - -jobs: - test_go_modules: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'recursive' - - - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest - id: filter_go - with: - packageLevelResolution: "modules" - github_token: ${{ secrets.github_token }} - timeout: "1m" - - - name: Run tests - run: go test -v ${{ steps.filter_go.outputs.changed_modules }} -``` - -This workflow will run tests for all changed Go modules and their dependencies whenever a pull request is opened or synchronized. - From 4a316fd83654b7cbfdc94ff2568952787780eedc Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 3 Apr 2024 07:46:45 -0600 Subject: [PATCH 36/39] feat: enable workflow input to define dependency level resolution --- contrib/git-changes-action/main.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/contrib/git-changes-action/main.go b/contrib/git-changes-action/main.go index ad63817c64..b2fda796dd 100644 --- a/contrib/git-changes-action/main.go +++ b/contrib/git-changes-action/main.go @@ -10,6 +10,7 @@ import ( "strings" "time" + moduledetector "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/module" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/package" "github.com/synapsecns/sanguine/contrib/git-changes-action/detector/tree" @@ -21,6 +22,7 @@ const defaultTimeout = "1m" func main() { token := githubactions.GetInput("github_token") + dependencyLevelResolution := githubactions.GetInput("dependencyLevelResolution") workingDirectory, err := os.Getwd() if err != nil { @@ -43,12 +45,12 @@ func main() { panic(err) } - noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false) + noDepChanged, noDepUnchanged, err := outputModuleChanges(workingDirectory, ct, false, dependencyLevelResolution) if err != nil { panic(err) } - depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true) + depChanged, depUnchanged, err := outputModuleChanges(workingDirectory, ct, true, dependencyLevelResolution) if err != nil { panic(err) } @@ -63,8 +65,15 @@ func main() { // outputModuleChanges outputs the changed modules. // this wraps detector.DetectChangedModules and handles the output formatting to be parsable by github actions. // the final output is a json array of strings. -func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool) (changedJSON string, unchangedJson string, err error) { - modules, err := packagedetector.DetectChangedModules(workingDirectory, ct, includeDeps) +func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool, dependencyLevelResolution string) (changedJSON string, unchangedJson string, err error) { + var modules map[string]bool + + if (dependencyLevelResolution == "packages") { + modules, err = packagedetector.DetectChangedModules(workingDirectory, ct, includeDeps) + } else { + modules, err = moduledetector.DetectChangedModules(workingDirectory, ct, includeDeps) + } + if err != nil { return changedJSON, unchangedJson, fmt.Errorf("failed to detect changed modules w/ include deps set to %v: %w", includeDeps, err) } From 1e99cd588f3983247fbbdf4c22a8e33fa86dc4c6 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 3 Apr 2024 07:50:21 -0600 Subject: [PATCH 37/39] Update README.md --- contrib/git-changes-action/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/git-changes-action/README.md b/contrib/git-changes-action/README.md index 100db6a378..85b8e1ffb9 100644 --- a/contrib/git-changes-action/README.md +++ b/contrib/git-changes-action/README.md @@ -9,7 +9,7 @@ This GitHub Action exports a variable that contains the list of Go modules chang 1. To use this action, add the following steps to your workflow: - Check out the current pull request using the actions/checkout action. It's recommended to set fetch-depth to 0 and submodules to recursive to ensure that all necessary dependencies are fetched. + Check out the current pull request using the actions/checkout action. It's recommended to set fetch-depth to 0 and submodules to 'recursive' to ensure that all necessary dependencies are fetched. ```yaml steps: @@ -17,7 +17,7 @@ This GitHub Action exports a variable that contains the list of Go modules chang with: fetch-depth: 0 submodules: 'recursive' - packageOfLevelResolution: "modules" + dependencyLevelResolution: "modules" ``` 1. Use the synapsecns/sanguine/git-changes-action Docker image to run the git-changes script, which exports a variable that contains the list of changed Go modules. @@ -28,14 +28,15 @@ This GitHub Action exports a variable that contains the list of Go modules chang with: github_token: ${{ secrets.github_token }} timeout: "1m" # optional, defaults to 1m - dependencyLevelResolution: "modules" or "packages" + dependencyLevelResolution: "modules" + # For package level resolution, use "packages" instead of "modules" ``` You can customize the behavior of the git-changes script by using the following inputs: - `github_token`: The token to use for authentication with the GitHub API. This is required to fetch information about the current pull request. - `timeout`: The maximum time to wait for the GitHub API to respond. Defaults to 1 minute. - - `dependencyLevelResolution`: "modules" or "packages" + - `dependencyLevelResolution`: "modules" or "packages". The output of the git-changes script is a comma-separated list of Go module paths. You can access this list using the `filter_go` output variable, like so: From 39b33bf0cb9e3958a4667de10cc6fe11b560c73b Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:47:58 -0600 Subject: [PATCH 38/39] fix: lint issues --- contrib/git-changes-action/main.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/git-changes-action/main.go b/contrib/git-changes-action/main.go index b2fda796dd..76f4123e47 100644 --- a/contrib/git-changes-action/main.go +++ b/contrib/git-changes-action/main.go @@ -22,7 +22,7 @@ const defaultTimeout = "1m" func main() { token := githubactions.GetInput("github_token") - dependencyLevelResolution := githubactions.GetInput("dependencyLevelResolution") + dependencyLevelResolution := githubactions.GetInput("dependencyLevelResolution") workingDirectory, err := os.Getwd() if err != nil { @@ -65,17 +65,17 @@ func main() { // outputModuleChanges outputs the changed modules. // this wraps detector.DetectChangedModules and handles the output formatting to be parsable by github actions. // the final output is a json array of strings. -func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool, dependencyLevelResolution string) (changedJSON string, unchangedJson string, err error) { - var modules map[string]bool +func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool, dependencyLevelResolution string) (changedJSON string, unchangedJSON string, err error) { + var modules map[string]bool - if (dependencyLevelResolution == "packages") { - modules, err = packagedetector.DetectChangedModules(workingDirectory, ct, includeDeps) - } else { - modules, err = moduledetector.DetectChangedModules(workingDirectory, ct, includeDeps) - } + if dependencyLevelResolution == "packages" { + modules, err = packagedetector.DetectChangedModules(workingDirectory, ct, includeDeps) + } else { + modules, err = moduledetector.DetectChangedModules(workingDirectory, ct, includeDeps) + } if err != nil { - return changedJSON, unchangedJson, fmt.Errorf("failed to detect changed modules w/ include deps set to %v: %w", includeDeps, err) + return changedJSON, unchangedJSON, fmt.Errorf("failed to detect changed modules w/ include deps set to %v: %w", includeDeps, err) } var changedModules, unchangedModules []string From c3ac016ec44787aa3e5146fd5f027be6e6dada80 Mon Sep 17 00:00:00 2001 From: oliver <8559757+oliverqx@users.noreply.github.com> Date: Wed, 3 Apr 2024 11:27:09 -0600 Subject: [PATCH 39/39] fix: update variable names --- contrib/git-changes-action/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/git-changes-action/main.go b/contrib/git-changes-action/main.go index 76f4123e47..a4a21ec34a 100644 --- a/contrib/git-changes-action/main.go +++ b/contrib/git-changes-action/main.go @@ -94,12 +94,12 @@ func outputModuleChanges(workingDirectory string, ct tree.Tree, includeDeps bool marshalledChanged, err := json.Marshal(changedModules) if err != nil { - return changedJSON, unchangedJson, fmt.Errorf("failed to marshall changed module json w/ include deps set to %v: %w", includeDeps, err) + return changedJSON, unchangedJSON, fmt.Errorf("failed to marshall changed module json w/ include deps set to %v: %w", includeDeps, err) } marshalledUnchanged, err := json.Marshal(unchangedModules) if err != nil { - return changedJSON, unchangedJson, fmt.Errorf("failed to marshall unchanged module json w/ include deps set to %v: %w", includeDeps, err) + return changedJSON, unchangedJSON, fmt.Errorf("failed to marshall unchanged module json w/ include deps set to %v: %w", includeDeps, err) } return string(marshalledChanged), string(marshalledUnchanged), nil