feat: support submodules for github integration (#3531)

api/vcs/v1/vcs.proto

@@ -37,6 +37,8 @@ message GetFileRequest {
   string ref = 2;
   // the path to the file as provided by the symbols
   string localPath = 3;
+  // the root path where the project lives inside the repository
+  string rootPath = 4;
 }
 
 message GetFileResponse {

examples/language-sdk-instrumentation/golang-push/rideshare/rideshare/rideshare.go

@@ -100,8 +100,11 @@ func ReadConfig() Config {
 		UseDebugTracer: os.Getenv("DEBUG_TRACER") == "1",
 		UseDebugLogger: os.Getenv("DEBUG_LOGGER") == "1",
 		Tags: map[string]string{
-			"region":   os.Getenv("REGION"),
-			"hostname": hostname,
+			"region":             os.Getenv("REGION"),
+			"hostname":           hostname,
+			"service_git_ref":    "HEAD",
+			"service_repository": "https://github.com/grafana/pyroscope",
+			"service_root_path":  "examples/language-sdk-instrumentation/golang-push/rideshare",
 		},
 
 		ParametersPoolSize: envIntOrDefault("PARAMETERS_POOL_SIZE", 1000),

pkg/distributor/distributor_test.go

@@ -1405,6 +1405,7 @@ func TestInjectMappingVersions(t *testing.T) {
 			Labels: []*typesv1.LabelPair{
 				{Name: phlaremodel.LabelNameServiceRepository, Value: "grafana/pyroscope"},
 				{Name: phlaremodel.LabelNameServiceGitRef, Value: "foobar"},
+				{Name: phlaremodel.LabelNameServiceRootPath, Value: "some-path"},
 			},
 			Samples: []*distributormodel.ProfileSample{
 				{
@@ -1418,6 +1419,7 @@ func TestInjectMappingVersions(t *testing.T) {
 			Labels: []*typesv1.LabelPair{
 				{Name: phlaremodel.LabelNameServiceRepository, Value: "grafana/pyroscope"},
 				{Name: phlaremodel.LabelNameServiceGitRef, Value: "foobar"},
+				{Name: phlaremodel.LabelNameServiceRootPath, Value: "some-path"},
 			},
 			Samples: []*distributormodel.ProfileSample{
 				{
@@ -1433,8 +1435,8 @@ func TestInjectMappingVersions(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "", in[0].Samples[0].Profile.StringTable[in[0].Samples[0].Profile.Mapping[0].BuildId])
 	require.Equal(t, `{"repository":"grafana/pyroscope"}`, in[1].Samples[0].Profile.StringTable[in[1].Samples[0].Profile.Mapping[0].BuildId])
-	require.Equal(t, `{"repository":"grafana/pyroscope","git_ref":"foobar"}`, in[2].Samples[0].Profile.StringTable[in[2].Samples[0].Profile.Mapping[0].BuildId])
-	require.Equal(t, `{"repository":"grafana/pyroscope","git_ref":"foobar","build_id":"foo"}`, in[3].Samples[0].Profile.StringTable[in[3].Samples[0].Profile.Mapping[0].BuildId])
+	require.Equal(t, `{"repository":"grafana/pyroscope","git_ref":"foobar","root_path":"some-path"}`, in[2].Samples[0].Profile.StringTable[in[2].Samples[0].Profile.Mapping[0].BuildId])
+	require.Equal(t, `{"repository":"grafana/pyroscope","git_ref":"foobar","build_id":"foo","root_path":"some-path"}`, in[3].Samples[0].Profile.StringTable[in[3].Samples[0].Profile.Mapping[0].BuildId])
 }
 
 func uncompressedProfileSize(t *testing.T, req *pushv1.PushRequest) int {

pkg/model/labels.go

@@ -34,6 +34,7 @@ const (
 	LabelNameServiceGitRef     = "service_git_ref"
 	LabelNameServiceName       = "service_name"
 	LabelNameServiceRepository = "service_repository"
+	LabelNameServiceRootPath   = "service_root_path"
 
 	LabelNameOrder     = "__order__"
 	LabelOrderEnforced = "enforced"
@@ -495,15 +496,18 @@ type ServiceVersion struct {
 	Repository string `json:"repository,omitempty"`
 	GitRef     string `json:"git_ref,omitempty"`
 	BuildID    string `json:"build_id,omitempty"`
+	RootPath   string `json:"root_path,omitempty"`
 }
 
 // ServiceVersionFromLabels Attempts to extract a service version from the given labels.
 // Returns false if no service version was found.
 func ServiceVersionFromLabels(lbls Labels) (ServiceVersion, bool) {
 	repo := lbls.Get(LabelNameServiceRepository)
 	gitref := lbls.Get(LabelNameServiceGitRef)
+	rootPath := lbls.Get(LabelNameServiceRootPath)
 	return ServiceVersion{
 		Repository: repo,
 		GitRef:     gitref,
-	}, repo != "" || gitref != ""
+		RootPath:   rootPath,
+	}, repo != "" || gitref != "" || rootPath != ""
 }

pkg/model/labels_test.go

@@ -317,3 +317,61 @@ func TestInsert(t *testing.T) {
 		})
 	}
 }
+
+func Test_ServiceVersionFromLabels(t *testing.T) {
+	tests := []struct {
+		name            string
+		input           Labels
+		expectedVersion ServiceVersion
+		expectedOk      bool
+	}{
+		{
+			name: "all present",
+			input: Labels{
+				{Name: LabelNameServiceRepository, Value: "repo"},
+				{Name: LabelNameServiceGitRef, Value: "ref"},
+				{Name: LabelNameServiceRootPath, Value: "some-path"},
+				{Name: "any-other-label", Value: "any-value"},
+			},
+			expectedVersion: ServiceVersion{
+				Repository: "repo",
+				GitRef:     "ref",
+				RootPath:   "some-path",
+			},
+			expectedOk: true,
+		},
+		{
+			name: "some present",
+			input: Labels{
+				{Name: LabelNameServiceRepository, Value: "repo"},
+				{Name: LabelNameServiceRootPath, Value: "some-path"},
+				{Name: "any-other-label", Value: "any-value"},
+			},
+			expectedVersion: ServiceVersion{
+				Repository: "repo",
+				GitRef:     "",
+				RootPath:   "some-path",
+			},
+			expectedOk: true,
+		},
+		{
+			name: "none present",
+			input: Labels{
+				{Name: "any-label", Value: "any-value"},
+			},
+			expectedVersion: ServiceVersion{
+				Repository: "",
+				GitRef:     "",
+				RootPath:   "",
+			},
+			expectedOk: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			version, ok := ServiceVersionFromLabels(test.input)
+			assert.Equal(t, test.expectedVersion, version)
+			assert.Equal(t, test.expectedOk, ok)
+		})
+	}
+}

pkg/querier/vcs/service.go

@@ -150,6 +150,7 @@ func (q *Service) GetFile(ctx context.Context, req *connect.Request[vcsv1.GetFil
 		ghClient,
 		gitURL,
 		req.Msg.LocalPath,
+		req.Msg.RootPath,
 		req.Msg.Ref,
 		http.DefaultClient,
 		log.With(q.logger, "repo", gitURL.GetRepoName()),

pkg/querier/vcs/source/find.go

@@ -22,16 +22,16 @@ type VCSClient interface {
 
 // FileFinder finds a file in a vcs repository.
 type FileFinder struct {
-	path, ref string
-	repo      giturl.IGitURL
+	path, ref, rootPath string
+	repo                giturl.IGitURL
 
 	client     VCSClient
 	httpClient *http.Client
 	logger     log.Logger
 }
 
 // NewFileFinder returns a new FileFinder.
-func NewFileFinder(client VCSClient, repo giturl.IGitURL, path, ref string, httpClient *http.Client, logger log.Logger) *FileFinder {
+func NewFileFinder(client VCSClient, repo giturl.IGitURL, path, rootPath, ref string, httpClient *http.Client, logger log.Logger) *FileFinder {
 	if ref == "" {
 		ref = "HEAD"
 	}
@@ -40,6 +40,7 @@ func NewFileFinder(client VCSClient, repo giturl.IGitURL, path, ref string, http
 		logger:     logger,
 		repo:       repo,
 		path:       path,
+		rootPath:   rootPath,
 		ref:        ref,
 		httpClient: httpClient,
 	}

pkg/querier/vcs/source/find_go.go

@@ -99,16 +99,15 @@ func (ff FileFinder) fetchGoogleSourceDependencyFile(ctx context.Context, mod go
 	return ff.fetchURL(ctx, url, true)
 }
 
-// tryFindGoFile tries to find the go file in the repo.
-// It tries to find the file in the repo by removing path segment after path segment.
+// tryFindGoFile tries to find the go file in the repo, under the rootPath.
+// It tries to find the file in the rootPath inside the repo by removing path segment after path segment.
 // maxAttempts is the maximum number of attempts to try to find the file in case the file path is very long.
-// For example, if the path is "github.com/grafana/grafana/pkg/infra/log/log.go", it will try to find the file at:
-// - github.com/grafana/grafana/pkg/infra/log/log.go
-// - grafana/grafana/pkg/infra/log/log.go
-// - pkg/infra/log/log.go
-// - infra/log/log.go
-// - log/log.go
-// - log.go
+// For example, if the path is "github.com/grafana/grafana/pkg/infra/log/log.go" and rootPath is "path/to/module1", it
+// will try to find the file at:
+// - "path/to/module1/pkg/infra/log/log.go"
+// - "path/to/module1/infra/log/log.go"
+// - "path/to/module1/log/log.go"
+// - "path/to/module1/log.go"
 func (ff FileFinder) tryFindGoFile(ctx context.Context, maxAttempts int) (*vcsv1.GetFileResponse, error) {
 	if maxAttempts <= 0 {
 		return nil, errors.New("invalid max attempts")
@@ -121,7 +120,7 @@ func (ff FileFinder) tryFindGoFile(ctx context.Context, maxAttempts int) (*vcsv1
 		content, err := ff.client.GetFile(ctx, client.FileRequest{
 			Owner: ff.repo.GetOwnerName(),
 			Repo:  ff.repo.GetRepoName(),
-			Path:  path,
+			Path:  strings.Join([]string{ff.rootPath, path}, "/"),
 			Ref:   ff.ref,
 		})
 		attempts++

pkg/querier/vcs/source/find_go_test.go

@@ -0,0 +1,94 @@
+package source
+
+import (
+	"context"
+	"testing"
+
+	giturl "github.com/kubescape/go-git-url"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/grafana/pyroscope/pkg/querier/vcs/client"
+)
+
+type VCSClientMock struct {
+	fileToFind       string
+	searchedSequence []string
+}
+
+func (c *VCSClientMock) GetFile(ctx context.Context, req client.FileRequest) (client.File, error) {
+	c.searchedSequence = append(c.searchedSequence, req.Path)
+	if req.Path == c.fileToFind {
+		return client.File{}, nil
+	} else {
+		return client.File{}, client.ErrNotFound
+	}
+}
+
+func Test_tryFindGoFile(t *testing.T) {
+	pyroscopeRepo, _ := giturl.NewGitURL("http://github.com/grafana/pyroscope")
+	tests := []struct {
+		name                  string
+		searchedPath          string
+		rootPath              string
+		repo                  giturl.IGitURL
+		clientMock            *VCSClientMock
+		attempts              int
+		expectedSearchedPaths []string
+		expectedError         error
+	}{
+		{
+			name:                  "happy case in root path",
+			searchedPath:          "/var/service1/src/main.go",
+			rootPath:              "",
+			repo:                  pyroscopeRepo,
+			clientMock:            &VCSClientMock{fileToFind: "/main.go"},
+			attempts:              5,
+			expectedSearchedPaths: []string{"/var/service1/src/main.go", "/service1/src/main.go", "/src/main.go", "/main.go"},
+			expectedError:         nil,
+		},
+		{
+			name:                  "happy case in submodule",
+			searchedPath:          "/src/main.go",
+			rootPath:              "service/example",
+			repo:                  pyroscopeRepo,
+			clientMock:            &VCSClientMock{fileToFind: "service/example/main.go"},
+			attempts:              5,
+			expectedSearchedPaths: []string{"service/example/src/main.go", "service/example/main.go"},
+			expectedError:         nil,
+		},
+		{
+			name:                  "path with repository preffix",
+			searchedPath:          "github.com/grafana/pyroscope/main.go",
+			rootPath:              "",
+			repo:                  pyroscopeRepo,
+			clientMock:            &VCSClientMock{fileToFind: "/main.go"},
+			attempts:              1,
+			expectedSearchedPaths: []string{"/main.go"},
+			expectedError:         nil,
+		},
+		{
+			name:                  "not found, attempts exceeded",
+			searchedPath:          "/var/service1/src/main.go",
+			rootPath:              "",
+			repo:                  pyroscopeRepo,
+			clientMock:            &VCSClientMock{fileToFind: "/main.go"},
+			attempts:              3,
+			expectedSearchedPaths: []string{"/var/service1/src/main.go", "/service1/src/main.go", "/src/main.go"},
+			expectedError:         client.ErrNotFound,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctxMock := context.Context(nil)
+			sut := FileFinder{
+				path:     tt.searchedPath,
+				rootPath: tt.rootPath,
+				repo:     tt.repo,
+				client:   tt.clientMock,
+			}
+			_, err := sut.tryFindGoFile(ctxMock, tt.attempts)
+			assert.Equal(t, tt.expectedSearchedPaths, (*tt.clientMock).searchedSequence)
+			assert.Equal(t, tt.expectedError, err)
+		})
+	}
+}