Merge branch 'kcl-lang:main' into patch-graph

README.md

@@ -201,6 +201,10 @@ For more information about [OCI registry support](./docs/kpm_oci.md).
 
 - A: `go install` will install the binary file to `$GOPATH/bin` by default. You need to add `$GOPATH/bin` to the environment variable `PATH`.
 
+## Contributing
+
+- See [contribution guideline](https://kcl-lang.io/docs/community/contribute/).
+
 ## Learn More
 
 - [OCI registry support](./docs/kpm_oci-zh.md).

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dominikbraun/graph v0.23.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fatih/color v1.10.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect

go.sum

@@ -113,6 +113,8 @@ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHz
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
+github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
+github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=

kpm.go

@@ -25,6 +25,7 @@ func main() {
 	app.UsageText = "kpm  <command> [arguments]..."
 	app.Commands = []*cli.Command{
 		cmd.NewInitCmd(kpmcli),
+		cmd.NewGraphCmd(kpmcli),
 		cmd.NewAddCmd(kpmcli),
 		cmd.NewPkgCmd(kpmcli),
 		cmd.NewMetadataCmd(kpmcli),

pkg/client/client.go

@@ -9,13 +9,15 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/dominikbraun/graph"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/otiai10/copy"
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"kcl-lang.io/kpm/pkg/constants"
 	"kcl-lang.io/kpm/pkg/env"
 	"kcl-lang.io/kpm/pkg/errors"
 	"kcl-lang.io/kpm/pkg/git"
+	pkgGraph "kcl-lang.io/kpm/pkg/graph"
 	"kcl-lang.io/kpm/pkg/oci"
 	"kcl-lang.io/kpm/pkg/opt"
 	pkg "kcl-lang.io/kpm/pkg/package"
@@ -439,6 +441,33 @@ func (c *KpmClient) CompileTarPkg(tarPath string, opts *opt.CompileOptions) (*kc
 	return c.CompileWithOpts(opts)
 }
 
+// CompileGitPkg will compile the kcl package from the git url.
+func (c *KpmClient) CompileGitPkg(gitOpts *git.CloneOptions, compileOpts *opt.CompileOptions) (*kcl.KCLResultList, error) {
+	// 1. Create the temporary directory to pull the tar.
+	tmpDir, err := os.MkdirTemp("", "")
+	if err != nil {
+		return nil, reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, please contact us to fix it.")
+	}
+	// clean the temp dir.
+	defer os.RemoveAll(tmpDir)
+
+	// 2. clone the git repo
+	_, err = git.CloneWithOpts(
+		git.WithCommit(gitOpts.Commit),
+		git.WithBranch(gitOpts.Branch),
+		git.WithTag(gitOpts.Tag),
+		git.WithRepoURL(gitOpts.RepoURL),
+		git.WithLocalPath(tmpDir),
+	)
+	if err != nil {
+		return nil, reporter.NewErrorEvent(reporter.FailedGetPkg, err, "failed to get the git repository")
+	}
+
+	compileOpts.SetPkgPath(tmpDir)
+
+	return c.CompileWithOpts(compileOpts)
+}
+
 // CompileOciPkg will compile the kcl package from the OCI reference or url.
 func (c *KpmClient) CompileOciPkg(ociSource, version string, opts *opt.CompileOptions) (*kcl.KCLResultList, error) {
 	ociOpts, err := c.ParseOciOptionFromString(ociSource, version)
@@ -567,7 +596,7 @@ func (c *KpmClient) AddDepToPkg(kclPkg *pkg.KclPkg, d *pkg.Dependency) error {
 	}
 
 	// download all the dependencies.
-	changedDeps, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies)
+	changedDeps, _, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies)
 
 	if err != nil {
 		return err
@@ -1097,6 +1126,36 @@ func (c *KpmClient) ParseOciOptionFromString(oci string, tag string) (*opt.OciOp
 	return ociOpt, nil
 }
 
+// GetDependencyGraph will get the dependency graph of kcl package dependencies
+func (c *KpmClient) GetDependencyGraph(kclPkg *pkg.KclPkg) (graph.Graph[string, string], error) {
+	_, depGraph, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies)
+	if err != nil {
+		return nil, err
+	}
+
+	sources, err := pkgGraph.FindSources(depGraph)
+	if err != nil {
+		return nil, err
+	}
+
+	// add the root vertex(package name) to the dependency graph.
+	root := fmt.Sprintf("%s@%s", kclPkg.GetPkgName(), kclPkg.GetPkgVersion())
+	err = depGraph.AddVertex(root)
+	if err != nil {
+		return nil, err
+	}
+
+	// make an edge between the root vertex and all the sources of the dependency graph.
+	for _, source := range sources {
+		err = depGraph.AddEdge(root, source)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return depGraph, nil
+}
+
 // dependencyExists will check whether the dependency exists in the local filesystem.
 func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependencies) *pkg.Dependency {
 
@@ -1121,15 +1180,15 @@ func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependen
 }
 
 // downloadDeps will download all the dependencies of the current kcl package.
-func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies) (*pkg.Dependencies, error) {
+func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencies) (*pkg.Dependencies, graph.Graph[string, string], error) {
 	newDeps := pkg.Dependencies{
 		Deps: make(map[string]pkg.Dependency),
 	}
 
 	// Traverse all dependencies in kcl.mod
 	for _, d := range deps.Deps {
 		if len(d.Name) == 0 {
-			return nil, errors.InvalidDependency
+			return nil, nil, errors.InvalidDependency
 		}
 
 		existDep := c.dependencyExists(&d, &lockDeps)
@@ -1141,7 +1200,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 		expectedSum := lockDeps.Deps[d.Name].Sum
 		// Clean the cache
 		if len(c.homePath) == 0 || len(d.FullName) == 0 {
-			return nil, errors.InternalBug
+			return nil, nil, errors.InternalBug
 		}
 		dir := filepath.Join(c.homePath, d.FullName)
 		os.RemoveAll(dir)
@@ -1150,15 +1209,15 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 
 		lockedDep, err := c.Download(&d, dir)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		if !lockedDep.IsFromLocal() {
 			if !c.noSumCheck && expectedSum != "" &&
 				lockedDep.Sum != expectedSum &&
 				existDep != nil &&
 				existDep.FullName == d.FullName {
-				return nil, reporter.NewErrorEvent(
+				return nil, nil, reporter.NewErrorEvent(
 					reporter.CheckSumMismatch,
 					errors.CheckSumMismatchError,
 					fmt.Sprintf("checksum for '%s' changed in lock file", lockedDep.Name),
@@ -1171,6 +1230,8 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 		lockDeps.Deps[d.Name] = *lockedDep
 	}
 
+	depGraph := graph.New(graph.StringHash, graph.Directed())
+
 	// Recursively download the dependencies of the new dependencies.
 	for _, d := range newDeps.Deps {
 		// Load kcl.mod file of the new downloaded dependencies.
@@ -1183,13 +1244,37 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 			if os.IsNotExist(err) {
 				continue
 			}
-			return nil, err
+			return nil, nil, err
 		}
 
 		// Download the dependencies.
-		nested, err := c.downloadDeps(deppkg.ModFile.Dependencies, lockDeps)
+		nested, nestedDepGraph, err := c.downloadDeps(deppkg.ModFile.Dependencies, lockDeps)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
+		}
+
+		source := fmt.Sprintf("%s@%s", d.Name, d.Version)
+		err = depGraph.AddVertex(source)
+		if err != nil && err != graph.ErrVertexAlreadyExists {
+			return nil, nil, err
+		}
+
+		sourcesOfNestedDepGraph, err := pkgGraph.FindSources(nestedDepGraph)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		depGraph, err = pkgGraph.Union(depGraph, nestedDepGraph)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// make an edge between the source of all nested dep graph and main dep graph
+		for _, sourceOfNestedDepGraph := range sourcesOfNestedDepGraph {
+			err = depGraph.AddEdge(source, sourceOfNestedDepGraph)
+			if err != nil && err != graph.ErrEdgeAlreadyExists {
+				return nil, nil, err
+			}
 		}
 
 		// Update kcl.mod.
@@ -1200,7 +1285,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 		}
 	}
 
-	return &newDeps, nil
+	return &newDeps, depGraph, nil
 }
 
 // pullTarFromOci will pull a kcl package tar file from oci registry.

pkg/client/client_test.go

@@ -12,9 +12,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/dominikbraun/graph"
 	"github.com/otiai10/copy"
 	"github.com/stretchr/testify/assert"
 	"kcl-lang.io/kpm/pkg/env"
+	"kcl-lang.io/kpm/pkg/git"
 	"kcl-lang.io/kpm/pkg/opt"
 	pkg "kcl-lang.io/kpm/pkg/package"
 	"kcl-lang.io/kpm/pkg/runner"
@@ -116,6 +118,45 @@ func TestDownloadLatestOci(t *testing.T) {
 	assert.Equal(t, err, nil)
 }
 
+func TestDependencyGraph(t *testing.T) {
+	testDir := getTestDir("test_dependency_graph")
+	assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl.mod.lock")), false)
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+	kclPkg, err := kpmcli.LoadPkgFromPath(testDir)
+	assert.Equal(t, err, nil)
+
+	depGraph, err := kpmcli.GetDependencyGraph(kclPkg)
+	assert.Equal(t, err, nil)
+	adjMap, err := depGraph.AdjacencyMap()
+	assert.Equal(t, err, nil)
+
+	edgeProp := graph.EdgeProperties{
+		Attributes: map[string]string{},
+		Weight:     0,
+		Data:       nil,
+	}
+	assert.Equal(t, adjMap,
+		map[string]map[string]graph.Edge[string]{
+			"dependency_graph@0.0.1": {
+				"teleport@0.1.0": {Source: "dependency_graph@0.0.1", Target: "teleport@0.1.0", Properties: edgeProp},
+				"rabbitmq@0.0.1": {Source: "dependency_graph@0.0.1", Target: "rabbitmq@0.0.1", Properties: edgeProp},
+				"agent@0.1.0":    {Source: "dependency_graph@0.0.1", Target: "agent@0.1.0", Properties: edgeProp},
+			},
+			"teleport@0.1.0": {
+				"k8s@1.28": {Source: "teleport@0.1.0", Target: "k8s@1.28", Properties: edgeProp},
+			},
+			"rabbitmq@0.0.1": {
+				"k8s@1.28": {Source: "rabbitmq@0.0.1", Target: "k8s@1.28", Properties: edgeProp},
+			},
+			"agent@0.1.0": {
+				"k8s@1.28": {Source: "agent@0.1.0", Target: "k8s@1.28", Properties: edgeProp},
+			},
+			"k8s@1.28": {},
+		},
+	)
+}
+
 func TestInitEmptyPkg(t *testing.T) {
 	testDir := initTestDir("test_init_empty_mod")
 	kclPkg := pkg.NewKclPkg(&opt.InitOptions{Name: "test_name", InitPath: testDir})
@@ -844,6 +885,29 @@ func TestRunWithNoSumCheck(t *testing.T) {
 	}()
 }
 
+func TestRemoteRun(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	opts := opt.DefaultCompileOptions()
+	gitOpts := git.NewCloneOptions("https://github.com/KusionStack/catalog", "", "0.1.2", "", "", nil)
+
+	opts.SetEntries([]string{"models/samples/hellocollaset/prod/main.k"})
+	result, err := kpmcli.CompileGitPkg(gitOpts, opts)
+	assert.Equal(t, err, nil)
+	assert.Equal(t, result.GetRawJsonResult(), "[{\"hellocollaset\": {\"workload\": {\"containers\": {\"nginx\": {\"image\": \"nginx:v2\"}}}}}]")
+
+	opts.SetEntries([]string{"models/samples/pgadmin/base/base.k"})
+	result, err = kpmcli.CompileGitPkg(gitOpts, opts)
+	assert.Equal(t, err, nil)
+	assert.Equal(t, result.GetRawJsonResult(), "[{\"pgadmin\": {\"workload\": {\"containers\": {\"pgadmin\": {\"image\": \"dpage/pgadmin4:latest\", \"env\": {\"PGADMIN_DEFAULT_EMAIL\": \"admin@admin.com\", \"PGADMIN_DEFAULT_PASSWORD\": \"secret://pgadmin-secret/pgadmin-default-password\", \"PGADMIN_PORT\": \"80\"}, \"resources\": {\"cpu\": \"500m\", \"memory\": \"512Mi\"}}}, \"secrets\": {\"pgadmin-secret\": {\"type\": \"opaque\", \"data\": {\"pgadmin-default-password\": \"*******\"}}}, \"replicas\": 1, \"ports\": [{\"port\": 80, \"protocol\": \"TCP\", \"public\": false}]}, \"database\": {\"pgadmin\": {\"type\": \"cloud\", \"version\": \"14.0\"}}}}]")
+
+	opts.SetEntries([]string{"models/samples/wordpress/prod/main.k"})
+	result, err = kpmcli.CompileGitPkg(gitOpts, opts)
+	assert.Equal(t, err, nil)
+	assert.Equal(t, result.GetRawJsonResult(), "[{\"wordpress\": {\"workload\": {\"containers\": {\"wordpress\": {\"image\": \"wordpress:6.3\", \"env\": {\"WORDPRESS_DB_HOST\": \"$(KUSION_DB_HOST_WORDPRESS)\", \"WORDPRESS_DB_USER\": \"$(KUSION_DB_USERNAME_WORDPRESS)\", \"WORDPRESS_DB_PASSWORD\": \"$(KUSION_DB_PASSWORD_WORDPRESS)\", \"WORDPRESS_DB_NAME\": \"mysql\"}, \"resources\": {\"cpu\": \"500m\", \"memory\": \"512Mi\"}}}, \"replicas\": 1, \"ports\": [{\"port\": 80, \"protocol\": \"TCP\", \"public\": false}]}, \"database\": {\"wordpress\": {\"type\": \"cloud\", \"version\": \"8.0\"}}}}]")
+}
+
 func TestUpdateWithNoSumCheck(t *testing.T) {
 	pkgPath := getTestDir("test_update_no_sum_check")
 	kpmcli, err := NewKpmClient()

pkg/client/test_data/test_dependency_graph/kcl.mod

@@ -0,0 +1,9 @@
+[package]
+name = "dependency_graph"
+edition = "0.0.1"
+version = "0.0.1"
+
+[dependencies]
+rabbitmq = "0.0.1"
+teleport = "0.1.0"
+agent = "0.1.0"