From 34002b84083c36c209b31b5cdce7948bd37ad4ca Mon Sep 17 00:00:00 2001 From: yueluhuan Date: Mon, 9 Sep 2024 11:03:26 +0800 Subject: [PATCH] Switch from ADAL to AzIdentity, Add Azure Storage Token Refresh, and Update Golang Version (#1977) Uses the new Azure Identity package for current support. Support automated Azure storage token refresh. Sets the default value of Athens storage account key and Azure managed identity resource id should be empty. --- Makefile | 2 +- config.dev.toml | 8 ++-- go.mod | 22 +++++---- go.sum | 42 ++++++++++------- pkg/config/azureblob.go | 2 +- pkg/config/config_test.go | 6 +-- pkg/stash/with_azureblob.go | 30 ++++++++++--- pkg/stash/with_azureblob_test.go | 6 +-- pkg/storage/azureblob/azureblob.go | 60 ++++++++++++++++++++++--- pkg/storage/azureblob/azureblob_test.go | 6 +-- 10 files changed, 135 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index ba3f64b88..63ecb520c 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ DATE=$(shell date -u +%Y-%m-%d-%H:%M:%S-%Z) GOLANGCI_LINT_VERSION=v1.51.2 ifndef GOLANG_VERSION -override GOLANG_VERSION = 1.20 +override GOLANG_VERSION = 1.22.4 endif .PHONY: build diff --git a/config.dev.toml b/config.dev.toml index d45409d2e..976af56df 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -534,15 +534,15 @@ ShutdownTimeout = 60 # Account Key to use with the storage account # Env override: ATHENS_AZURE_ACCOUNT_KEY - AccountKey = "MY_AZURE_BLOB_ACCOUNT_KEY" + AccountKey = "" # Managed Identity Resource Id to use with the storage account # Env override: ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID - ManagedIdentityResourceId = "MY_AZURE_MANAGED_IDENTITY_RESOURCE_ID" + ManagedIdentityResourceId = "" # Storage Resource to use with the storage account - # Env override: ATHENS_AZURE_STORAGE_RESOURCE - StorageResource = "MY_AZURE_STORAGE_RESOURCE" + # Env override: ATHENS_AZURE_CREDENTIAL_SCOPE + CredentialScope = "" # Name of container in the blob storage # Env override: ATHENS_AZURE_CONTAINER_NAME diff --git a/go.mod b/go.mod index 6f9997f25..194890562 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,9 @@ require ( contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.1.0 contrib.go.opencensus.io/exporter/stackdriver v0.6.0 + github.com/Azure/azure-pipeline-go v0.2.2 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 github.com/Azure/azure-storage-blob-go v0.10.0 github.com/Azure/go-autorest/autorest/adal v0.9.23 github.com/BurntSushi/toml v1.0.0 @@ -43,9 +46,9 @@ require ( go.etcd.io/etcd/server/v3 v3.5.12 go.mongodb.org/mongo-driver v1.7.1 go.opencensus.io v0.24.0 - golang.org/x/mod v0.8.0 + golang.org/x/mod v0.17.0 golang.org/x/oauth2 v0.16.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.7.0 google.golang.org/api v0.162.0 ) @@ -56,11 +59,12 @@ require ( cloud.google.com/go/iam v1.1.6 // indirect cloud.google.com/go/monitoring v1.18.0 // indirect cloud.google.com/go/trace v1.10.5 // indirect - github.com/Azure/azure-pipeline-go v0.2.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 // indirect github.com/agext/levenshtein v1.2.1 // indirect github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect @@ -96,6 +100,7 @@ require ( github.com/go-stack/stack v1.8.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -127,13 +132,14 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/philhofer/fwd v1.0.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.47.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tinylib/msgp v1.0.2 // indirect @@ -161,10 +167,10 @@ require ( go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.19.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect diff --git a/go.sum b/go.sum index 64faabf19..d734438f7 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,12 @@ contrib.go.opencensus.io/exporter/stackdriver v0.6.0/go.mod h1:QeFzMJDAw8TXt5+aR dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg= github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0SzsJiXQVRGsRXxs= github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -83,6 +89,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -249,6 +257,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -455,6 +465,8 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -479,8 +491,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -619,8 +631,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -655,8 +667,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -701,8 +713,8 @@ golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -726,8 +738,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -784,8 +796,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -802,8 +814,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -866,7 +878,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/config/azureblob.go b/pkg/config/azureblob.go index c0f38f397..454a3a6ee 100644 --- a/pkg/config/azureblob.go +++ b/pkg/config/azureblob.go @@ -5,6 +5,6 @@ type AzureBlobConfig struct { AccountName string `envconfig:"ATHENS_AZURE_ACCOUNT_NAME" validate:"required"` AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY"` ManagedIdentityResourceID string `envconfig:"ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID"` - StorageResource string `envconfig:"ATHENS_AZURE_STORAGE_RESOURCE"` + CredentialScope string `envconfig:"ATHENS_AZURE_CREDENTIAL_SCOPE"` ContainerName string `envconfig:"ATHENS_AZURE_CONTAINER_NAME" validate:"required"` } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 985410746..2fad9c5e5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -236,9 +236,9 @@ func TestParseExampleConfig(t *testing.T) { }, AzureBlob: &AzureBlobConfig{ AccountName: "MY_AZURE_BLOB_ACCOUNT_NAME", - AccountKey: "MY_AZURE_BLOB_ACCOUNT_KEY", - ManagedIdentityResourceID: "MY_AZURE_MANAGED_IDENTITY_RESOURCE_ID", - StorageResource: "MY_AZURE_STORAGE_RESOURCE", + AccountKey: "", + ManagedIdentityResourceID: "", + CredentialScope: "", ContainerName: "MY_AZURE_BLOB_CONTAINER_NAME", }, External: &External{URL: ""}, diff --git a/pkg/stash/with_azureblob.go b/pkg/stash/with_azureblob.go index 1717dd231..ef0c28964 100644 --- a/pkg/stash/with_azureblob.go +++ b/pkg/stash/with_azureblob.go @@ -8,12 +8,14 @@ import ( "net/url" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/Azure/go-autorest/autorest/adal" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/storage" + "github.com/gomods/athens/pkg/storage/azureblob" "github.com/google/uuid" ) @@ -22,7 +24,7 @@ import ( func WithAzureBlobLock(conf *config.AzureBlobConfig, timeout time.Duration, checker storage.Checker) (Wrapper, error) { const op errors.Op = "stash.WithAzureBlobLock" - if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.StorageResource == "") { + if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.CredentialScope == "") { return nil, errors.E(op, "either account key or managed identity resource id and storage resource must be set") } accountURL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName)) @@ -37,15 +39,33 @@ func WithAzureBlobLock(conf *config.AzureBlobConfig, timeout time.Duration, chec } } if conf.ManagedIdentityResourceID != "" { - spStorageToken, err := adal.NewServicePrincipalTokenFromManagedIdentity(conf.StorageResource, &adal.ManagedIdentityOptions{IdentityResourceID: conf.ManagedIdentityResourceID}) + msiCred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ResourceID(conf.ManagedIdentityResourceID), + }) if err != nil { return nil, errors.E(op, err) } - err = spStorageToken.Refresh() + token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{Scopes: []string{conf.CredentialScope}}) if err != nil { return nil, errors.E(op, err) } - cred = azblob.NewTokenCredential(spStorageToken.OAuthToken(), nil) + cred = azblob.NewTokenCredential(token.Token, func(tc azblob.TokenCredential) time.Duration { + fmt.Printf("refreshing token started at: %s", time.Now()) + refreshedToken, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{conf.CredentialScope}, + }) + if err != nil { + fmt.Printf("error getting token: %s during token refresh process", err) + // token refresh may fail due to transient errors, so we return a non-zero duration + // to retry the token refresh after a short delay + return time.Minute + } + tc.SetToken(refreshedToken.Token) + + refreshDuration := time.Until(refreshedToken.ExpiresOn.Add(-azureblob.TokenRefreshTolerance)) + fmt.Printf("refresh duration: %s", refreshDuration) + return refreshDuration + }) } pipe := azblob.NewPipeline(cred, azblob.PipelineOptions{}) serviceURL := azblob.NewServiceURL(*accountURL, pipe) diff --git a/pkg/stash/with_azureblob_test.go b/pkg/stash/with_azureblob_test.go index 805af369f..124ede39f 100644 --- a/pkg/stash/with_azureblob_test.go +++ b/pkg/stash/with_azureblob_test.go @@ -87,8 +87,8 @@ func (ms *mockAzureBlobStasher) Stash(ctx context.Context, mod, ver string) (str func getAzureTestConfig(containerName string) *config.AzureBlobConfig { key := os.Getenv("ATHENS_AZURE_ACCOUNT_KEY") resourceId := os.Getenv("ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID") - storageResource := os.Getenv("ATHENS_AZURE_STORAGE_RESOURCE") - if key == "" && (resourceId == "" || storageResource == "") { + credentialScope := os.Getenv("ATHENS_AZURE_CREDENTIAL_SCOPE") + if key == "" && (resourceId == "" || credentialScope == "") { return nil } name := os.Getenv("ATHENS_AZURE_ACCOUNT_NAME") @@ -99,7 +99,7 @@ func getAzureTestConfig(containerName string) *config.AzureBlobConfig { AccountName: name, AccountKey: key, ManagedIdentityResourceID: resourceId, - StorageResource: storageResource, + CredentialScope: credentialScope, ContainerName: containerName, } } diff --git a/pkg/storage/azureblob/azureblob.go b/pkg/storage/azureblob/azureblob.go index 53712d5b3..a3f8de024 100644 --- a/pkg/storage/azureblob/azureblob.go +++ b/pkg/storage/azureblob/azureblob.go @@ -8,6 +8,9 @@ import ( "net/url" "time" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" @@ -19,13 +22,55 @@ type azureBlobStoreClient struct { containerURL *azblob.ContainerURL } -func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, containerName string) (*azureBlobStoreClient, error) { +const ( + // TokenRefreshTolerance defines the duration before the token's actual expiration time + // during which the token should be refreshed. This helps ensure that the token is + // refreshed in a timely manner, avoiding potential issues with token expiration. + TokenRefreshTolerance = 5 * time.Minute +) + +func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, credScope, managedIdentityResourceID, containerName string) (*azureBlobStoreClient, error) { const op errors.Op = "azureblob.newBlobStoreClient" - cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) - if err != nil { - return nil, errors.E(op, err) + var pipe pipeline.Pipeline + if managedIdentityResourceID != "" { + msiCred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ResourceID(managedIdentityResourceID), + }) + if err != nil { + return nil, errors.E(op, err) + } + token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{credScope}, + }) + if err != nil { + return nil, errors.E(op, err) + } + tokenCred := azblob.NewTokenCredential(token.Token, func(tc azblob.TokenCredential) time.Duration { + fmt.Printf("refreshing token started at: %s", time.Now()) + refreshedToken, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{credScope}, + }) + if err != nil { + fmt.Printf("error getting token: %s during token refresh process", err) + // token refresh may fail due to transient errors, so we return a non-zero duration + // to retry the token refresh after a short delay + return time.Minute + } + tc.SetToken(refreshedToken.Token) + + refreshDuration := time.Until(refreshedToken.ExpiresOn.Add(-TokenRefreshTolerance)) + fmt.Printf("refresh duration: %s", refreshDuration) + return refreshDuration + }) + pipe = azblob.NewPipeline(tokenCred, azblob.PipelineOptions{}) + } + if pipe == nil && accountKey != "" { + cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + return nil, errors.E(op, err) + } + pipe = azblob.NewPipeline(cred, azblob.PipelineOptions{}) } - pipe := azblob.NewPipeline(cred, azblob.PipelineOptions{}) serviceURL := azblob.NewServiceURL(*accountURL, pipe) // rules on container names: // https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names @@ -50,7 +95,10 @@ func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) if err != nil { return nil, errors.E(op, err) } - cl, err := newBlobStoreClient(u, conf.AccountName, conf.AccountKey, conf.ContainerName) + if conf.AccountKey == "" && (conf.ManagedIdentityResourceID == "" || conf.CredentialScope == "") { + return nil, errors.E(op, "either account key or managed identity resource id and storage resource must be set") + } + cl, err := newBlobStoreClient(u, conf.AccountName, conf.AccountKey, conf.CredentialScope, conf.ManagedIdentityResourceID, conf.ContainerName) if err != nil { return nil, errors.E(op, err) } diff --git a/pkg/storage/azureblob/azureblob_test.go b/pkg/storage/azureblob/azureblob_test.go index 6f6c34a95..62edeb49e 100644 --- a/pkg/storage/azureblob/azureblob_test.go +++ b/pkg/storage/azureblob/azureblob_test.go @@ -71,8 +71,8 @@ func getStorage(t testing.TB) *Storage { func getTestConfig(containerName string) *config.AzureBlobConfig { key := os.Getenv("ATHENS_AZURE_ACCOUNT_KEY") resourceId := os.Getenv("ATHENS_AZURE_MANAGED_IDENTITY_RESOURCE_ID") - storageResource := os.Getenv("ATHENS_AZURE_STORAGE_RESOURCE") - if key == "" && (resourceId == "" || storageResource == "") { + credentialScope := os.Getenv("ATHENS_AZURE_CREDENTIAL_SCOPE") + if key == "" && (resourceId == "" || credentialScope == "") { return nil } name := os.Getenv("ATHENS_AZURE_ACCOUNT_NAME") @@ -83,7 +83,7 @@ func getTestConfig(containerName string) *config.AzureBlobConfig { AccountName: name, AccountKey: key, ManagedIdentityResourceID: resourceId, - StorageResource: storageResource, + CredentialScope: credentialScope, ContainerName: containerName, } }