feat: add option authType for s3 storage

config.sample.yaml

@@ -171,6 +171,8 @@ datastores:
       accessSecret: ""
       ssl: true
       bucketName: "your-media-bucket"
+      # Available auth types: static, env, iam, default to static
+      #authType: static
       # An optional region for where this S3 endpoint is located. Typically not needed, though
       # some providers will need this (like Scaleway). Uncomment to use.
       #region: "sfo2"
@@ -597,4 +599,4 @@ sentry:
   environment: ""
 
   # Whether or not to turn on sentry's built in debugging. This will increase log output.
-  debug: false
+  debug: false

go.mod

@@ -37,7 +37,7 @@ require (
 	github.com/kettek/apng v0.0.0-20220622131934-4e92eded13be
 	github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f
 	github.com/lib/pq v1.10.6
-	github.com/minio/minio-go/v6 v6.0.57
+	github.com/minio/minio-go/v7 v7.0.47
 	github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
@@ -231,6 +231,7 @@ require (
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/raulk/go-watchdog v1.3.0 // indirect
+	github.com/rs/xid v1.4.0 // indirect
 	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect

go.sum

@@ -812,7 +812,6 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -840,7 +839,6 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
 github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -1319,11 +1317,10 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo
 github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc=
 github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
-github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw=
-github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM=
+github.com/minio/minio-go/v7 v7.0.47 h1:sLiuCKGSIcn/MI6lREmTzX91DX/oRau4ia0j6e6eOSs=
+github.com/minio/minio-go/v7 v7.0.47/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
@@ -1576,6 +1573,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM=
 github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk=
 github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A=
@@ -1618,7 +1617,6 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED
 github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -2318,7 +2316,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=

storage/datastore/ds_s3/s3_store.go

@@ -1,14 +1,15 @@
 package ds_s3
 
 import (
+	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/turt2live/matrix-media-repo/common/config"
@@ -21,6 +22,7 @@ import (
 var stores = make(map[string]*s3Datastore)
 
 type s3Datastore struct {
+	ctx          context.Context
 	conf         config.DatastoreConfig
 	dsId         string
 	client       *minio.Client
@@ -37,35 +39,50 @@ func GetOrCreateS3Datastore(dsId string, conf config.DatastoreConfig) (*s3Datast
 
 	endpoint, epFound := conf.Options["endpoint"]
 	bucket, bucketFound := conf.Options["bucketName"]
+	authType, authTypeFound := conf.Options["authType"]
 	accessKeyId, keyFound := conf.Options["accessKeyId"]
 	accessSecret, secretFound := conf.Options["accessSecret"]
-	region, regionFound := conf.Options["region"]
+	region := conf.Options["region"]
 	tempPath, tempPathFound := conf.Options["tempPath"]
 	storageClass, storageClassFound := conf.Options["storageClass"]
-	if !epFound || !bucketFound || !keyFound || !secretFound {
-		return nil, errors.New("invalid configuration: missing s3 options")
+	if !epFound || !bucketFound {
+		return nil, errors.New("invalid configuration: missing s3 endpoint/bucket")
 	}
 	if !tempPathFound {
 		logrus.Warn("Datastore ", dsId, " (s3) does not have a tempPath set - this could lead to excessive memory usage by the media repo")
 	}
 	if !storageClassFound {
 		storageClass = "STANDARD"
 	}
-
-	useSsl := true
-	useSslStr, sslFound := conf.Options["ssl"]
-	if sslFound && useSslStr != "" {
-		useSsl, _ = strconv.ParseBool(useSslStr)
+	if !authTypeFound {
+		authType = "static"
 	}
 
-	var s3client *minio.Client
-	var err error
+	useSSL := true
+	useSSLStr, sslFound := conf.Options["ssl"]
+	if sslFound && useSSLStr != "" {
+		useSSL, _ = strconv.ParseBool(useSSLStr)
+	}
 
-	if regionFound {
-		s3client, err = minio.NewWithRegion(endpoint, accessKeyId, accessSecret, useSsl, region)
-	} else {
-		s3client, err = minio.New(endpoint, accessKeyId, accessSecret, useSsl)
+	var cred *credentials.Credentials
+	switch authType {
+	case "static":
+		if !keyFound || !secretFound {
+			return nil, errors.New("invalid configuration: missing s3 key/secret")
+		}
+		cred = credentials.NewStaticV4(accessKeyId, accessSecret, "")
+	case "env":
+		cred = credentials.NewEnvAWS()
+	case "iam":
+		cred = credentials.NewIAM("")
+	default:
+		return nil, errors.New("invalid configuration: unsupported s3 auth type")
 	}
+	s3client, err := minio.New(endpoint, &minio.Options{
+		Creds:  cred,
+		Region: region,
+		Secure: useSSL,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -109,7 +126,7 @@ func ParseS3URL(s3url string) (string, string, string, error) {
 }
 
 func (s *s3Datastore) EnsureBucketExists() error {
-	found, err := s.client.BucketExists(s.bucket)
+	found, err := s.client.BucketExists(s.ctx, s.bucket)
 	if err != nil {
 		return err
 	}
@@ -144,26 +161,27 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx r
 	defer close(done)
 
 	var hash string
-	var sizeBytes int64
 	var hashErr error
 	var uploadErr error
+	var uploadInfo minio.UploadInfo
 
 	go func() {
 		defer ws3.Close()
 		ctx.Log.Info("Calculating hash of stream...")
-		hash, hashErr = util.GetSha256HashOfStream(ioutil.NopCloser(tr))
+		hash, hashErr = util.GetSha256HashOfStream(io.NopCloser(tr))
 		ctx.Log.Info("Hash of file is ", hash)
 		done <- true
 	}()
 
+	uploadOpts := minio.PutObjectOptions{StorageClass: s.storageClass}
 	go func() {
 		if expectedLength <= 0 {
 			if s.tempPath != "" {
 				ctx.Log.Info("Buffering file to temp path due to unknown file size")
 				var f *os.File
-				f, uploadErr = ioutil.TempFile(s.tempPath, "mr*")
+				f, uploadErr = os.CreateTemp(s.tempPath, "mr*")
 				if uploadErr != nil {
-					io.Copy(ioutil.Discard, rs3)
+					io.Copy(io.Discard, rs3)
 					done <- true
 					return
 				}
@@ -183,8 +201,8 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx r
 			}
 		}
 		ctx.Log.Info("Uploading file...")
-		sizeBytes, uploadErr = s.client.PutObjectWithContext(ctx, s.bucket, objectName, rs3, expectedLength, minio.PutObjectOptions{StorageClass: s.storageClass})
-		ctx.Log.Info("Uploaded ", sizeBytes, " bytes to s3")
+		uploadInfo, uploadErr = s.client.PutObject(ctx, s.bucket, objectName, rs3, expectedLength, uploadOpts)
+		ctx.Log.Info("Uploaded ", uploadInfo.Size, " bytes to s3")
 		done <- true
 	}()
 
@@ -195,7 +213,7 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx r
 	obj := &types.ObjectInfo{
 		Location:   objectName,
 		Sha256Hash: hash,
-		SizeBytes:  sizeBytes,
+		SizeBytes:  uploadInfo.Size,
 	}
 
 	if hashErr != nil {
@@ -212,16 +230,19 @@ func (s *s3Datastore) UploadFile(file io.ReadCloser, expectedLength int64, ctx r
 
 func (s *s3Datastore) DeleteObject(location string) error {
 	logrus.Info("Deleting object from bucket ", s.bucket, ": ", location)
-	return s.client.RemoveObject(s.bucket, location)
+	opts := minio.RemoveObjectOptions{}
+	return s.client.RemoveObject(s.ctx, s.bucket, location, opts)
 }
 
 func (s *s3Datastore) DownloadObject(location string) (io.ReadCloser, error) {
 	logrus.Info("Downloading object from bucket ", s.bucket, ": ", location)
-	return s.client.GetObject(s.bucket, location, minio.GetObjectOptions{})
+	opts := minio.GetObjectOptions{}
+	return s.client.GetObject(s.ctx, s.bucket, location, opts)
 }
 
 func (s *s3Datastore) ObjectExists(location string) bool {
-	stat, err := s.client.StatObject(s.bucket, location, minio.StatObjectOptions{})
+	opts := minio.StatObjectOptions{}
+	stat, err := s.client.StatObject(s.ctx, s.bucket, location, opts)
 	if err != nil {
 		return false
 	}
@@ -230,15 +251,15 @@ func (s *s3Datastore) ObjectExists(location string) bool {
 
 func (s *s3Datastore) OverwriteObject(location string, stream io.ReadCloser) error {
 	defer cleanup.DumpAndCloseStream(stream)
-	_, err := s.client.PutObject(s.bucket, location, stream, -1, minio.PutObjectOptions{StorageClass: s.storageClass})
+	opts := minio.PutObjectOptions{StorageClass: s.storageClass}
+	_, err := s.client.PutObject(s.ctx, s.bucket, location, stream, -1, opts)
 	return err
 }
 
 func (s *s3Datastore) ListObjects() ([]string, error) {
-	doneCh := make(chan struct{})
-	defer close(doneCh)
 	list := make([]string, 0)
-	for message := range s.client.ListObjectsV2(s.bucket, "", true, doneCh) {
+	opts := minio.ListObjectsOptions{Recursive: true}
+	for message := range s.client.ListObjects(s.ctx, s.bucket, opts) {
 		list = append(list, message.Key)
 	}
 	return list, nil