From 7d0fc3d142e721b32a8c0572fea681eaf60e9100 Mon Sep 17 00:00:00 2001 From: yueluhuan Date: Mon, 5 Aug 2024 10:29:07 +0000 Subject: [PATCH] update config ATHENS_AZURE_CREDENTIAL_SCOPE --- cmd/proxy/actions/app_proxy.go | 7 ------- config.dev.toml | 4 ++-- pkg/config/azureblob.go | 2 +- pkg/config/config.go | 5 ----- pkg/config/config_test.go | 2 +- pkg/download/mode/mode.go | 6 ------ pkg/stash/with_azureblob.go | 16 ++-------------- pkg/stash/with_azureblob_test.go | 6 +++--- pkg/storage/azureblob/azureblob.go | 20 ++++---------------- pkg/storage/azureblob/azureblob_test.go | 6 +++--- 10 files changed, 16 insertions(+), 58 deletions(-) diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index d162c64d9..0117bc9db 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -41,7 +41,6 @@ func addProxyRoutes( indexer, err := getIndex(c) if err != nil { - l.Println("error creating indexer") return err } r.HandleFunc("/index", indexHandler(indexer)) @@ -49,11 +48,9 @@ func addProxyRoutes( for _, sumdb := range c.SumDBs { sumdbURL, err := url.Parse(sumdb) if err != nil { - l.Println("error parsing sumdb URL") return err } if sumdbURL.Scheme != "https" { - l.Println("sumdb must have an https scheme") return fmt.Errorf("sumdb: %v must have an https scheme", sumdb) } supportPath := path.Join("/sumdb", sumdbURL.Host, "/supported") @@ -95,12 +92,10 @@ func addProxyRoutes( c.GoBinaryEnvVars.Add("GONOSUMDB", strings.Join(c.NoSumPatterns, ",")) } if err := c.GoBinaryEnvVars.Validate(); err != nil { - l.Println("error validating GoBinaryEnvVars") return err } mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs) if err != nil { - l.Println("error creating module fetcher") return err } @@ -108,14 +103,12 @@ func addProxyRoutes( checker := storage.WithChecker(s) withSingleFlight, err := getSingleFlight(l, c, s, checker) if err != nil { - l.Println("error creating single flight") return err } st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight) df, err := mode.NewFile(c.DownloadMode, c.DownloadURL) if err != nil { - l.Println("error creating download file") return err } diff --git a/config.dev.toml b/config.dev.toml index 6ffb71cda..976af56df 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -541,8 +541,8 @@ ShutdownTimeout = 60 ManagedIdentityResourceId = "" # Storage Resource to use with the storage account - # Env override: ATHENS_AZURE_STORAGE_RESOURCE - StorageResource = "" + # Env override: ATHENS_AZURE_CREDENTIAL_SCOPE + CredentialScope = "" # Name of container in the blob storage # Env override: ATHENS_AZURE_CONTAINER_NAME diff --git a/pkg/config/azureblob.go b/pkg/config/azureblob.go index c0f38f397..454a3a6ee 100644 --- a/pkg/config/azureblob.go +++ b/pkg/config/azureblob.go @@ -5,6 +5,6 @@ type AzureBlobConfig struct { AccountName string `envconfig:"ATHENS_AZURE_ACCOUNT_NAME" validate:"required"` AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY"` ManagedIdentityResourceID string `envconfig:"ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID"` - StorageResource string `envconfig:"ATHENS_AZURE_STORAGE_RESOURCE"` + CredentialScope string `envconfig:"ATHENS_AZURE_CREDENTIAL_SCOPE"` ContainerName string `envconfig:"ATHENS_AZURE_CONTAINER_NAME" validate:"required"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index e81bbc049..b591b949a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -259,11 +259,6 @@ func envOverride(config *Config) error { if err != nil { return err } - - // TODO(yuelu) remove log after test is done - blobConfig := *config.Storage.AzureBlob - log.Printf("BlobConfig: %v", blobConfig) - portEnv := os.Getenv("PORT") // ATHENS_PORT takes precedence over PORT if portEnv != "" && os.Getenv("ATHENS_PORT") == "" { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index fd1eb5fe2..2fad9c5e5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -238,7 +238,7 @@ func TestParseExampleConfig(t *testing.T) { AccountName: "MY_AZURE_BLOB_ACCOUNT_NAME", AccountKey: "", ManagedIdentityResourceID: "", - StorageResource: "", + CredentialScope: "", ContainerName: "MY_AZURE_BLOB_CONTAINER_NAME", }, External: &External{URL: ""}, diff --git a/pkg/download/mode/mode.go b/pkg/download/mode/mode.go index 82e11a937..20960eda0 100644 --- a/pkg/download/mode/mode.go +++ b/pkg/download/mode/mode.go @@ -3,7 +3,6 @@ package mode import ( "encoding/base64" "fmt" - "log" "os" "path/filepath" "strings" @@ -67,13 +66,8 @@ func NewFile(m Mode, downloadURL string) (*DownloadFile, error) { } return parseFile(bts) } else if strings.HasPrefix(string(m), "custom:") { - // TODO(yuelu) remove this line after test passes - log.Printf("mode: %s", string(m[7:])) - bts, err := base64.StdEncoding.DecodeString(string(m[7:])) if err != nil { - // TODO(yuelu) remove this line after test passes - log.Println("[NewFile] error decoding base64 string") return nil, err } return parseFile(bts) diff --git a/pkg/stash/with_azureblob.go b/pkg/stash/with_azureblob.go index 3e04b6986..e7d0adf78 100644 --- a/pkg/stash/with_azureblob.go +++ b/pkg/stash/with_azureblob.go @@ -23,7 +23,7 @@ import ( func WithAzureBlobLock(conf *config.AzureBlobConfig, timeout time.Duration, checker storage.Checker) (Wrapper, error) { const op errors.Op = "stash.WithAzureBlobLock" - if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.StorageResource == "") { + if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.CredentialScope == "") { return nil, errors.E(op, "either account key or managed identity resource id and storage resource must be set") } accountURL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName)) @@ -38,25 +38,13 @@ func WithAzureBlobLock(conf *config.AzureBlobConfig, timeout time.Duration, chec } } if conf.ManagedIdentityResourceID != "" { - // spStorageToken, err := adal.NewServicePrincipalTokenFromManagedIdentity(conf.StorageResource, &adal.ManagedIdentityOptions{IdentityResourceID: conf.ManagedIdentityResourceID}) - // if err != nil { - // return nil, errors.E(op, err) - // } - // err = spStorageToken.Refresh() - // if err != nil { - // return nil, errors.E(op, err) - // } - // cred = azblob.NewTokenCredential(spStorageToken.OAuthToken(), nil) - msiCred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ ID: azidentity.ResourceID(conf.ManagedIdentityResourceID), }) if err != nil { return nil, errors.E(op, err) } - token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ - Scopes: []string{"https://management.azure.com/.default"}, - }) + token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{Scopes: []string{conf.CredentialScope}}) if err != nil { return nil, errors.E(op, err) } diff --git a/pkg/stash/with_azureblob_test.go b/pkg/stash/with_azureblob_test.go index 805af369f..124ede39f 100644 --- a/pkg/stash/with_azureblob_test.go +++ b/pkg/stash/with_azureblob_test.go @@ -87,8 +87,8 @@ func (ms *mockAzureBlobStasher) Stash(ctx context.Context, mod, ver string) (str func getAzureTestConfig(containerName string) *config.AzureBlobConfig { key := os.Getenv("ATHENS_AZURE_ACCOUNT_KEY") resourceId := os.Getenv("ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID") - storageResource := os.Getenv("ATHENS_AZURE_STORAGE_RESOURCE") - if key == "" && (resourceId == "" || storageResource == "") { + credentialScope := os.Getenv("ATHENS_AZURE_CREDENTIAL_SCOPE") + if key == "" && (resourceId == "" || credentialScope == "") { return nil } name := os.Getenv("ATHENS_AZURE_ACCOUNT_NAME") @@ -99,7 +99,7 @@ func getAzureTestConfig(containerName string) *config.AzureBlobConfig { AccountName: name, AccountKey: key, ManagedIdentityResourceID: resourceId, - StorageResource: storageResource, + CredentialScope: credentialScope, ContainerName: containerName, } } diff --git a/pkg/storage/azureblob/azureblob.go b/pkg/storage/azureblob/azureblob.go index 8951af2fd..93deaf4d5 100644 --- a/pkg/storage/azureblob/azureblob.go +++ b/pkg/storage/azureblob/azureblob.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "log" "net/http" "net/url" "time" @@ -24,19 +23,10 @@ type azureBlobStoreClient struct { containerURL *azblob.ContainerURL } -func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, storageResource, managedIdentityResourceID, containerName string) (*azureBlobStoreClient, error) { +func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, credScope, managedIdentityResourceID, containerName string) (*azureBlobStoreClient, error) { const op errors.Op = "azureblob.newBlobStoreClient" var pipe pipeline.Pipeline if managedIdentityResourceID != "" { - // spStorageToken, err := adal.NewServicePrincipalTokenFromManagedIdentity(storageResource, &adal.ManagedIdentityOptions{IdentityResourceID: managedIdentityResourceID}) - // if err != nil { - // return nil, errors.E(op, err) - // } - // err = spStorageToken.Refresh() - // if err != nil { - // return nil, errors.E(op, err) - // } - msiCred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ ID: azidentity.ResourceID(managedIdentityResourceID), }) @@ -44,13 +34,11 @@ func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, storageRes return nil, errors.E(op, err) } token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ - Scopes: []string{"https://management.azure.com/.default"}, + Scopes: []string{credScope}, }) if err != nil { return nil, errors.E(op, err) } - // TODO(yuelu): delete this when test passes - log.Println("token:", token) tokenCred := azblob.NewTokenCredential(token.Token, nil) pipe = azblob.NewPipeline(tokenCred, azblob.PipelineOptions{}) } @@ -85,10 +73,10 @@ func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) if err != nil { return nil, errors.E(op, err) } - if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.StorageResource == "") { + if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.CredentialScope == "") { return nil, errors.E(op, "either account key or managed identity resource id and storage resource must be set") } - cl, err := newBlobStoreClient(u, conf.AccountName, conf.AccountKey, conf.StorageResource, conf.ManagedIdentityResourceID, conf.ContainerName) + cl, err := newBlobStoreClient(u, conf.AccountName, conf.AccountKey, conf.CredentialScope, conf.ManagedIdentityResourceID, conf.ContainerName) if err != nil { return nil, errors.E(op, err) } diff --git a/pkg/storage/azureblob/azureblob_test.go b/pkg/storage/azureblob/azureblob_test.go index 6f6c34a95..62edeb49e 100644 --- a/pkg/storage/azureblob/azureblob_test.go +++ b/pkg/storage/azureblob/azureblob_test.go @@ -71,8 +71,8 @@ func getStorage(t testing.TB) *Storage { func getTestConfig(containerName string) *config.AzureBlobConfig { key := os.Getenv("ATHENS_AZURE_ACCOUNT_KEY") resourceId := os.Getenv("ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID") - storageResource := os.Getenv("ATHENS_AZURE_STORAGE_RESOURCE") - if key == "" && (resourceId == "" || storageResource == "") { + credentialScope := os.Getenv("ATHENS_AZURE_CREDENTIAL_SCOPE") + if key == "" && (resourceId == "" || credentialScope == "") { return nil } name := os.Getenv("ATHENS_AZURE_ACCOUNT_NAME") @@ -83,7 +83,7 @@ func getTestConfig(containerName string) *config.AzureBlobConfig { AccountName: name, AccountKey: key, ManagedIdentityResourceID: resourceId, - StorageResource: storageResource, + CredentialScope: credentialScope, ContainerName: containerName, } }