From d08446257e8393deb97a97598adc6e985d54f525 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 7 Nov 2022 09:57:00 +0000 Subject: [PATCH 1/4] Add new config option `usersMaxBytes` --- common/config/models_domain.go | 9 +++++---- config.sample.yaml | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/common/config/models_domain.go b/common/config/models_domain.go index 32427247..3887638c 100644 --- a/common/config/models_domain.go +++ b/common/config/models_domain.go @@ -17,10 +17,11 @@ type QuotasConfig struct { } type UploadsConfig struct { - MaxSizeBytes int64 `yaml:"maxBytes"` - MinSizeBytes int64 `yaml:"minBytes"` - ReportedMaxSizeBytes int64 `yaml:"reportedMaxBytes"` - Quota QuotasConfig `yaml:"quotas"` + MaxSizeBytes int64 `yaml:"maxBytes"` + UsersMaxSizeBytes []QuotaUserConfig `yaml:"quotas,flow"` + MinSizeBytes int64 `yaml:"minBytes"` + ReportedMaxSizeBytes int64 `yaml:"reportedMaxBytes"` + Quota QuotasConfig `yaml:"quotas"` } type DatastoreConfig struct { diff --git a/config.sample.yaml b/config.sample.yaml index fa49e74f..333cf53a 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -214,6 +214,10 @@ uploads: # The maximum individual file size a user can upload. maxBytes: 104857600 # 100MB default, 0 to disable + usersMaxBytes: + - glob: "@*:*" # Affect all users. Use asterisks (*) to match any character. + maxBytes: 104857600 # 100MB default, 0 to disable + # The minimum number of bytes to let people upload. This is recommended to be non-zero to # ensure that the "cost" of running the media repo is worthwhile - small file uploads tend # to waste more CPU and database resources than small files, thus a default of 100 bytes From e60211f9fa1b74514019b11fa7642319495d18d3 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 7 Nov 2022 09:58:12 +0000 Subject: [PATCH 2/4] Limit uploads per user --- api/r0/public_config.go | 11 ++--------- api/r0/upload.go | 5 +++-- .../preview_resource_handler.go | 5 +++-- .../upload_controller/upload_controller.go | 13 ++++++++----- quota/quota.go | 16 ++++++++++++++++ 5 files changed, 32 insertions(+), 18 deletions(-) diff --git a/api/r0/public_config.go b/api/r0/public_config.go index 66d313f2..d35dbb37 100644 --- a/api/r0/public_config.go +++ b/api/r0/public_config.go @@ -5,6 +5,7 @@ import ( "github.com/turt2live/matrix-media-repo/api" "github.com/turt2live/matrix-media-repo/common/rcontext" + "github.com/turt2live/matrix-media-repo/quota" ) type PublicConfigResponse struct { @@ -12,15 +13,7 @@ type PublicConfigResponse struct { } func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} { - uploadSize := rctx.Config.Uploads.ReportedMaxSizeBytes - if uploadSize == 0 { - uploadSize = rctx.Config.Uploads.MaxSizeBytes - } - - if uploadSize < 0 { - uploadSize = 0 // invokes the omitEmpty - } - + uploadSize := quota.GetUserUploadMaxSizeBytes(rctx, user.UserId) return &PublicConfigResponse{ UploadMaxSize: uploadSize, } diff --git a/api/r0/upload.go b/api/r0/upload.go index 63b0170b..089466c6 100644 --- a/api/r0/upload.go +++ b/api/r0/upload.go @@ -1,12 +1,13 @@ package r0 import ( - "github.com/getsentry/sentry-go" "io" "io/ioutil" "net/http" "path/filepath" + "github.com/getsentry/sentry-go" + "github.com/sirupsen/logrus" "github.com/turt2live/matrix-media-repo/api" "github.com/turt2live/matrix-media-repo/common" @@ -35,7 +36,7 @@ func UploadMedia(r *http.Request, rctx rcontext.RequestContext, user api.UserInf contentType = "application/octet-stream" // binary } - if upload_controller.IsRequestTooLarge(r.ContentLength, r.Header.Get("Content-Length"), rctx) { + if upload_controller.IsRequestTooLarge(r.ContentLength, r.Header.Get("Content-Length"), rctx, user.UserId) { io.Copy(ioutil.Discard, r.Body) // Ditch the entire request return api.RequestTooLarge() } diff --git a/controllers/preview_controller/preview_resource_handler.go b/controllers/preview_controller/preview_resource_handler.go index 86c42b9d..50995ca2 100644 --- a/controllers/preview_controller/preview_resource_handler.go +++ b/controllers/preview_controller/preview_resource_handler.go @@ -2,9 +2,10 @@ package preview_controller import ( "fmt" - "github.com/getsentry/sentry-go" "sync" + "github.com/getsentry/sentry-go" + "github.com/disintegration/imaging" "github.com/sirupsen/logrus" "github.com/turt2live/matrix-media-repo/common" @@ -129,7 +130,7 @@ func urlPreviewWorkFn(request *resource_handler.WorkRequest) (resp *urlPreviewRe } // Store the thumbnail, if there is one - if preview.Image != nil && !upload_controller.IsRequestTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader, ctx) { + if preview.Image != nil && !upload_controller.IsRequestTooLarge(preview.Image.ContentLength, preview.Image.ContentLengthHeader, ctx, info.forUserId) { contentLength := upload_controller.EstimateContentLength(preview.Image.ContentLength, preview.Image.ContentLengthHeader) // UploadMedia will close the read stream for the thumbnail and dedupe the image diff --git a/controllers/upload_controller/upload_controller.go b/controllers/upload_controller/upload_controller.go index 7936faa7..d9ea42b4 100644 --- a/controllers/upload_controller/upload_controller.go +++ b/controllers/upload_controller/upload_controller.go @@ -2,12 +2,13 @@ package upload_controller import ( "fmt" - "github.com/getsentry/sentry-go" "io" "io/ioutil" "strconv" "time" + "github.com/getsentry/sentry-go" + "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -15,6 +16,7 @@ import ( "github.com/turt2live/matrix-media-repo/common/rcontext" "github.com/turt2live/matrix-media-repo/internal_cache" "github.com/turt2live/matrix-media-repo/plugins" + "github.com/turt2live/matrix-media-repo/quota" "github.com/turt2live/matrix-media-repo/storage" "github.com/turt2live/matrix-media-repo/storage/datastore" "github.com/turt2live/matrix-media-repo/types" @@ -32,12 +34,13 @@ type AlreadyUploadedFile struct { ObjectInfo *types.ObjectInfo } -func IsRequestTooLarge(contentLength int64, contentLengthHeader string, ctx rcontext.RequestContext) bool { - if ctx.Config.Uploads.MaxSizeBytes <= 0 { +func IsRequestTooLarge(contentLength int64, contentLengthHeader string, ctx rcontext.RequestContext, userId string) bool { + maxSize := quota.GetUserUploadMaxSizeBytes(ctx, userId) + if maxSize <= 0 { return false } if contentLength >= 0 { - return contentLength > ctx.Config.Uploads.MaxSizeBytes + return contentLength > maxSize } if contentLengthHeader != "" { parsed, err := strconv.ParseInt(contentLengthHeader, 10, 64) @@ -47,7 +50,7 @@ func IsRequestTooLarge(contentLength int64, contentLengthHeader string, ctx rcon return true // Invalid header } - return parsed > ctx.Config.Uploads.MaxSizeBytes + return parsed > maxSize } return false // We can only assume diff --git a/quota/quota.go b/quota/quota.go index 1a8649ff..dd27a5c0 100644 --- a/quota/quota.go +++ b/quota/quota.go @@ -33,3 +33,19 @@ func IsUserWithinQuota(ctx rcontext.RequestContext, userId string) (bool, error) return true, nil // no rules == no quota } + +func GetUserUploadMaxSizeBytes(ctx rcontext.RequestContext, userId string) int64 { + var maxValue int64 = 0 + for _, q := range ctx.Config.Uploads.UsersMaxSizeBytes { + if glob.Glob(q.Glob, userId) { + if q.MaxBytes > maxValue { + maxValue = q.MaxBytes + } + } + } + if maxValue == 0 { + return ctx.Config.Uploads.MaxSizeBytes + } else { + return maxValue + } +} From e8304364d279f94f5dba344bf20f73601da7da54 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 7 Nov 2022 10:05:46 +0000 Subject: [PATCH 3/4] Use the **last** match --- config.sample.yaml | 5 ++++- quota/quota.go | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index 333cf53a..5253118b 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -214,9 +214,12 @@ uploads: # The maximum individual file size a user can upload. maxBytes: 104857600 # 100MB default, 0 to disable + # The maximum individual file size a user can upload. If a user does not match + # any of the globs in here, the global `maxBytes` value is used. + # If several globs match for a given user, the **last** matching value is used. usersMaxBytes: - glob: "@*:*" # Affect all users. Use asterisks (*) to match any character. - maxBytes: 104857600 # 100MB default, 0 to disable + maxBytes: 104857600 # maxBytes, by default. 0 to disable # The minimum number of bytes to let people upload. This is recommended to be non-zero to # ensure that the "cost" of running the media repo is worthwhile - small file uploads tend diff --git a/quota/quota.go b/quota/quota.go index dd27a5c0..c597bbdd 100644 --- a/quota/quota.go +++ b/quota/quota.go @@ -38,9 +38,8 @@ func GetUserUploadMaxSizeBytes(ctx rcontext.RequestContext, userId string) int64 var maxValue int64 = 0 for _, q := range ctx.Config.Uploads.UsersMaxSizeBytes { if glob.Glob(q.Glob, userId) { - if q.MaxBytes > maxValue { - maxValue = q.MaxBytes - } + // Use the *last* match + maxValue = q.MaxBytes } } if maxValue == 0 { From 10ede82e92801e5fbeff20311d402a94151af83d Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 7 Nov 2022 10:08:14 +0000 Subject: [PATCH 4/4] Use the `first` matching glob --- config.sample.yaml | 2 +- quota/quota.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index 5253118b..08bdc8c2 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -216,7 +216,7 @@ uploads: # The maximum individual file size a user can upload. If a user does not match # any of the globs in here, the global `maxBytes` value is used. - # If several globs match for a given user, the **last** matching value is used. + # The first matching glob is always used. usersMaxBytes: - glob: "@*:*" # Affect all users. Use asterisks (*) to match any character. maxBytes: 104857600 # maxBytes, by default. 0 to disable diff --git a/quota/quota.go b/quota/quota.go index c597bbdd..fd52887e 100644 --- a/quota/quota.go +++ b/quota/quota.go @@ -35,16 +35,16 @@ func IsUserWithinQuota(ctx rcontext.RequestContext, userId string) (bool, error) } func GetUserUploadMaxSizeBytes(ctx rcontext.RequestContext, userId string) int64 { - var maxValue int64 = 0 - for _, q := range ctx.Config.Uploads.UsersMaxSizeBytes { - if glob.Glob(q.Glob, userId) { - // Use the *last* match - maxValue = q.MaxBytes + for _, u := range ctx.Config.Uploads.UsersMaxSizeBytes { + if glob.Glob(u.Glob, userId) { + if u.MaxBytes == 0 { + return ctx.Config.Uploads.MaxSizeBytes + } else if u.MaxBytes < 0 { + return 0 + } else { + return u.MaxBytes + } } } - if maxValue == 0 { - return ctx.Config.Uploads.MaxSizeBytes - } else { - return maxValue - } + }