From 7c6de92f2a2df5f9a504fa537501c7dca371d3c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Mon, 26 Sep 2022 18:17:46 +0200 Subject: [PATCH 01/10] worker/aws: prefer bucket from `TargetOptions` if provided Flip the logic when deciding if to use the Bucket from the job or worker configuration. Previously, the Bucket from the worker configuration was always preferred if it was set, even if it was provided in the job itself. This made it impossible to override the configuration. Change the logic to use the Bucket from the worker configuration only if it was not set in the job. Report an error if no bucket name was provided with the job and there is also none specified in the configuration. --- cmd/osbuild-worker/jobimpl-osbuild.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index de8d0622ee..abaf05b530 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -121,8 +121,11 @@ func (impl *OSBuildJobImpl) getAWSForS3Target(options *target.AWSS3TargetOptions // Endpoint == "" && Region != "" => AWS (Weldr and Composer) if options.Endpoint == "" && options.Region != "" { aws, err = impl.getAWS(options.Region, options.AccessKeyID, options.SecretAccessKey, options.SessionToken) - if impl.AWSBucket != "" { + if bucket == "" { bucket = impl.AWSBucket + if bucket == "" { + err = fmt.Errorf("No AWS bucket provided") + } } } else if options.Endpoint != "" && options.Region != "" { // Endpoint != "" && Region != "" => Generic S3 Weldr API aws, err = impl.getAWSForS3TargetFromOptions(options) @@ -471,8 +474,12 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { } bucket := targetOptions.Bucket - if impl.AWSBucket != "" { + if bucket == "" { bucket = impl.AWSBucket + if bucket == "" { + targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "No AWS bucket provided", nil) + break + } } // TODO: Remove this once multiple exports will be supported and used by image definitions From 2d5fbf0d1e03168dec1be4ea3e3b446426ba8e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Mon, 3 Oct 2022 14:13:14 +0200 Subject: [PATCH 02/10] worker/aws: don't generate object key in worker There is a desire to make the worker as "dumb" as possible. Therefore it is not desired to generate the AWS object key names in the worker if it was not provided in the job. Modify the worker code to not generate the AWS object key in any case and instead set an error in case the object key was not provided. Modify Weldr API implementation to generate the object key, if it was not provided by the user. This is consistent with Cloud API implementation. --- cmd/osbuild-worker/jobimpl-osbuild.go | 15 ++++++++++----- internal/weldr/upload.go | 13 +++++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index abaf05b530..206f23427f 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -468,9 +468,9 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { break } - key := targetOptions.Key - if key == "" { - key = uuid.New().String() + if targetOptions.Key == "" { + targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "No AWS object key provided", nil) + break } bucket := targetOptions.Bucket @@ -496,13 +496,13 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { } } - _, err = a.Upload(imagePath, bucket, key) + _, err = a.Upload(imagePath, bucket, targetOptions.Key) if err != nil { targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, err.Error(), nil) break } - ami, err := a.Register(jobTarget.ImageName, bucket, key, targetOptions.ShareWithAccounts, common.CurrentArch()) + ami, err := a.Register(jobTarget.ImageName, bucket, targetOptions.Key, targetOptions.ShareWithAccounts, common.CurrentArch()) if err != nil { targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorImportingImage, err.Error(), nil) break @@ -525,6 +525,11 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { break } + if targetOptions.Key == "" { + targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "No AWS object key provided", nil) + break + } + url, targetError := uploadToS3(a, outputDirectory, jobTarget.OsbuildArtifact.ExportName, bucket, targetOptions.Key, jobTarget.OsbuildArtifact.ExportFilename, targetOptions.Public) if targetError != nil { targetResult.TargetError = targetError diff --git a/internal/weldr/upload.go b/internal/weldr/upload.go index df7e2a70bc..04ab1626a6 100644 --- a/internal/weldr/upload.go +++ b/internal/weldr/upload.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "fmt" "strings" "time" @@ -250,6 +251,10 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target. switch options := u.Settings.(type) { case *awsUploadSettings: + key := options.Key + if key == "" { + key = fmt.Sprintf("composer-api-%s", uuid.New().String()) + } t.Name = target.TargetNameAWS t.Options = &target.AWSTargetOptions{ Region: options.Region, @@ -257,9 +262,13 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target. SecretAccessKey: options.SecretAccessKey, SessionToken: options.SessionToken, Bucket: options.Bucket, - Key: options.Key, + Key: key, } case *awsS3UploadSettings: + key := options.Key + if key == "" { + key = fmt.Sprintf("composer-api-%s", uuid.New().String()) + } t.Name = target.TargetNameAWSS3 t.Options = &target.AWSS3TargetOptions{ Region: options.Region, @@ -267,7 +276,7 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target. SecretAccessKey: options.SecretAccessKey, SessionToken: options.SessionToken, Bucket: options.Bucket, - Key: options.Key, + Key: key, Endpoint: options.Endpoint, CABundle: options.CABundle, SkipSSLVerification: options.SkipSSLVerification, From 267ea123338a8e7d903007d13c443a1ab9698932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Fri, 23 Sep 2022 12:54:17 +0200 Subject: [PATCH 03/10] worker/osbuild: use dedicated struct for GCP config internally Previously, the internal `OSBuildJobImpl` structure defined only `GCPCreds` member. This is not practical, once there will be more than one GCP-related variable. Define a new `GCPConfiguration` structure, move the credentials variable into it and use it in `OSBuildJobImpl` instead. --- cmd/osbuild-worker/jobimpl-osbuild.go | 10 +++++++--- cmd/osbuild-worker/main.go | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index 206f23427f..20b2ebc69f 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -31,6 +31,10 @@ import ( "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" ) +type GCPConfiguration struct { + Creds string +} + type S3Configuration struct { Creds string Endpoint string @@ -52,7 +56,7 @@ type OSBuildJobImpl struct { Store string Output string KojiServers map[string]kojiServer - GCPCreds string + GCPConfig GCPConfiguration AzureCreds *azure.Credentials AWSCreds string AWSBucket string @@ -160,9 +164,9 @@ func (impl *OSBuildJobImpl) getGCP(credentials []byte) (*gcp.GCP, error) { if credentials != nil { logrus.Info("[GCP] ๐Ÿ”‘ using credentials provided with the job request") return gcp.New(credentials) - } else if impl.GCPCreds != "" { + } else if impl.GCPConfig.Creds != "" { logrus.Info("[GCP] ๐Ÿ”‘ using credentials from the worker configuration") - return gcp.NewFromFile(impl.GCPCreds) + return gcp.NewFromFile(impl.GCPConfig.Creds) } else { logrus.Info("[GCP] ๐Ÿ”‘ using Application Default Credentials via Google library") return gcp.New(nil) diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index 7c54786823..46dac35d6b 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -351,9 +351,9 @@ func main() { // If the credentials are not provided in the configuration, then the // worker will rely on the GCP library to authenticate using default means. - var gcpCredentials string + var gcpConfig GCPConfiguration if config.GCP != nil { - gcpCredentials = config.GCP.Credentials + gcpConfig.Creds = config.GCP.Credentials } // If the credentials are not provided in the configuration, then the @@ -435,7 +435,7 @@ func main() { Store: store, Output: output, KojiServers: kojiServers, - GCPCreds: gcpCredentials, + GCPConfig: gcpConfig, AzureCreds: azureCredentials, AWSCreds: awsCredentials, AWSBucket: awsBucket, From 6706a704a669e7c84b533e90dc7e28d5d454238d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Mon, 26 Sep 2022 18:30:49 +0200 Subject: [PATCH 04/10] worker/gcp: allow setting Bucket in worker configuration Extend the worker's configuration to allow setting GCP Bucket to use when uploading images to GCP. The value from the configuration is used only if not provided in the TargetOptions of the job. In GCP, the region of the bucket does not limit importing of the image to a particular region. So it is completely possible to use a single Bucket to import images to any and all regions. Return an error in case no bucket name was set in the job nor in the worker configuration. --- cmd/osbuild-worker/config.go | 1 + cmd/osbuild-worker/jobimpl-osbuild.go | 22 ++++++++++++++++------ cmd/osbuild-worker/main.go | 1 + 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/osbuild-worker/config.go b/cmd/osbuild-worker/config.go index d68a02e3b1..b968f687f2 100644 --- a/cmd/osbuild-worker/config.go +++ b/cmd/osbuild-worker/config.go @@ -23,6 +23,7 @@ type kojiServerConfig struct { type gcpConfig struct { Credentials string `toml:"credentials"` + Bucket string `toml:"bucket"` } type azureConfig struct { diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index 20b2ebc69f..adb920a186 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -32,7 +32,8 @@ import ( ) type GCPConfiguration struct { - Creds string + Creds string + Bucket string } type S3Configuration struct { @@ -579,9 +580,18 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { break } - logWithId.Infof("[GCP] ๐Ÿš€ Uploading image to: %s/%s", targetOptions.Bucket, targetOptions.Object) + bucket := targetOptions.Bucket + if bucket == "" { + bucket = impl.GCPConfig.Bucket + if bucket == "" { + targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "No GCP bucket provided", nil) + break + } + } + + logWithId.Infof("[GCP] ๐Ÿš€ Uploading image to: %s/%s", bucket, targetOptions.Object) _, err = g.StorageObjectUpload(ctx, path.Join(outputDirectory, jobTarget.OsbuildArtifact.ExportName, jobTarget.OsbuildArtifact.ExportFilename), - targetOptions.Bucket, targetOptions.Object, map[string]string{gcp.MetadataKeyImageName: jobTarget.ImageName}) + bucket, targetOptions.Object, map[string]string{gcp.MetadataKeyImageName: jobTarget.ImageName}) if err != nil { targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, err.Error(), nil) break @@ -589,14 +599,14 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { logWithId.Infof("[GCP] ๐Ÿ“ฅ Importing image into Compute Engine as '%s'", jobTarget.ImageName) - _, importErr := g.ComputeImageInsert(ctx, targetOptions.Bucket, targetOptions.Object, jobTarget.ImageName, []string{targetOptions.Region}, gcp.GuestOsFeaturesByDistro(targetOptions.Os)) + _, importErr := g.ComputeImageInsert(ctx, bucket, targetOptions.Object, jobTarget.ImageName, []string{targetOptions.Region}, gcp.GuestOsFeaturesByDistro(targetOptions.Os)) if importErr == nil { logWithId.Infof("[GCP] ๐ŸŽ‰ Image import finished successfully") } // Cleanup storage before checking for errors - logWithId.Infof("[GCP] ๐Ÿงน Deleting uploaded image file: %s/%s", targetOptions.Bucket, targetOptions.Object) - if err = g.StorageObjectDelete(ctx, targetOptions.Bucket, targetOptions.Object); err != nil { + logWithId.Infof("[GCP] ๐Ÿงน Deleting uploaded image file: %s/%s", bucket, targetOptions.Object) + if err = g.StorageObjectDelete(ctx, bucket, targetOptions.Object); err != nil { logWithId.Errorf("[GCP] Encountered error while deleting object: %v", err) } diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index 46dac35d6b..03e321c97a 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -354,6 +354,7 @@ func main() { var gcpConfig GCPConfiguration if config.GCP != nil { gcpConfig.Creds = config.GCP.Credentials + gcpConfig.Bucket = config.GCP.Bucket } // If the credentials are not provided in the configuration, then the From 10d60723b50dfd63fbf6ff49878c856db3bb78de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Mon, 26 Sep 2022 18:37:14 +0200 Subject: [PATCH 05/10] internal/target/gcp: make `Bucket` optional The Bucket can now be set also in the worker configuration. --- internal/target/gcp_target.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/target/gcp_target.go b/internal/target/gcp_target.go index 008591537b..3979f00e80 100644 --- a/internal/target/gcp_target.go +++ b/internal/target/gcp_target.go @@ -5,7 +5,7 @@ const TargetNameGCP TargetName = "org.osbuild.gcp" type GCPTargetOptions struct { Region string `json:"region"` Os string `json:"os"` // not exposed in cloudapi for now - Bucket string `json:"bucket"` + Bucket string `json:"bucket,omitempty"` Object string `json:"object"` ShareWithAccounts []string `json:"shareWithAccounts,omitempty"` From 6acb77ead0703156f6aa4962037e5cfdb66b06b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Tue, 27 Sep 2022 11:17:12 +0200 Subject: [PATCH 06/10] cloudapi: make `Bucket` optional in `GCPUploadOptions` GCP Bucket to use can be now configured in the worker configuration. Make the `Bucket` optional in the Cloud API when uploading image to GCP. Adjust the Cloud API test case to configure GCP Bucket on the worker and not provide it in the API request. --- internal/cloudapi/v2/handler.go | 6 +- internal/cloudapi/v2/openapi.v2.gen.go | 138 ++++++++++++------------- internal/cloudapi/v2/openapi.v2.yml | 1 - test/cases/api/gcp.sh | 1 - tools/provision.sh | 7 +- 5 files changed, 80 insertions(+), 73 deletions(-) diff --git a/internal/cloudapi/v2/handler.go b/internal/cloudapi/v2/handler.go index ea745bc512..b0b2310c46 100644 --- a/internal/cloudapi/v2/handler.go +++ b/internal/cloudapi/v2/handler.go @@ -448,10 +448,14 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error { } imageName := fmt.Sprintf("composer-api-%s", uuid.New().String()) + var bucket string + if gcpUploadOptions.Bucket != nil { + bucket = *gcpUploadOptions.Bucket + } t := target.NewGCPTarget(&target.GCPTargetOptions{ Region: gcpUploadOptions.Region, Os: imageType.Arch().Distro().Name(), // not exposed in cloudapi - Bucket: gcpUploadOptions.Bucket, + Bucket: bucket, // the uploaded object must have a valid extension Object: fmt.Sprintf("%s.tar.gz", imageName), ShareWithAccounts: share, diff --git a/internal/cloudapi/v2/openapi.v2.gen.go b/internal/cloudapi/v2/openapi.v2.gen.go index f6ca45469f..f1717d7de6 100644 --- a/internal/cloudapi/v2/openapi.v2.gen.go +++ b/internal/cloudapi/v2/openapi.v2.gen.go @@ -339,7 +339,7 @@ type Filesystem struct { // GCPUploadOptions defines model for GCPUploadOptions. type GCPUploadOptions struct { // Name of an existing STANDARD Storage class Bucket. - Bucket string `json:"bucket"` + Bucket *string `json:"bucket,omitempty"` // The name to use for the imported and shared Compute Engine image. // The image name must be unique within the GCP project, which is used @@ -836,74 +836,74 @@ var swaggerSpec = []string{ "EGBnWFlwThkIuJcGX6CNNfSvyKb0SzbAHDgAdb/fJ2nwUQcg3sPtrDLK48tA1/0XdF3uUpE1g05hnyhJ", "KmH0WW4E4w+zz5KuHRboDiY8kQc6dSAmp7/7fyVCpZ5g6GGBgP8W/OIy7EC2+hpHbts+QrW9km6JP/tQ", "BH13ObJRvS+AMvBlh6ZkrftYNDH3+/jGQQoqgGQ1JiF/t7XpN+U9ncakIrV2fEN5OHTyUumUP21xNsv9", - "oM/g6Ms/XfiUDpfJD9fYH5dMVK6+hP+ymyyCXENEh0RkJgxiPVPKlyqF0l6PIQIuvS83uRX0jVcoMc3C", - "AmnCYzvDWdaOX47L76/z/usDYqejlYtUxNPPNezrcz0cyVZqxNs72x+wN/NX+xfqHhTp3/a1YkVWUdZt", - "cWWH9G/hLLwnUSjcNxwcCV67v5+OhAcx5DUrDgOwnUtNDkDvDPNTwV2pkdgOfvqU+b/DKpogAhyTxYiE", - "RVDBhUQDFzxjwQyzPBw8RX5y6K4f33xi/IKa4CXSTZRZ57mCpzBYFLzAhAto2+qFqbn+vyEAUyrc2iKo", - "v1sdMBUb+P7DBrx83m3M4GINbs5d6fYlsqQbZHGSo6wbBT9HOmUw05QuXaYB+TtOqI0g3+lZzBfz+ZN8", - "NZtPdKwQm/vR902P0H+c0SnOGgpxYMOylJnqteVNttLiDCcHcvls14qWi+mEDMUcMR7Lx5X21+AF5G9Q", - "BdXAG4gbriTZ23UKO2HhkMtwkFchKt0a27ASXxz8lu+Bf8+OKB06hDtJwYtwE78NcoZJckwhLOqOMz7c", - "OMe/CCqgnfRphwsKaXpdDe4XYfud0+/u6dOpYOWIF7JChkjCnrgJidz1IKyCijDIaoNfAtadgnzxOF+e", - "FHV4jE4q5YleKk9qk1oR1koVVIHVql6cHOcNA35NS18MggmDRLMyNp4hwNbpnw08ZiE7V8v5K2tOmpCv", - "O1uVeItkpTTiaeb93d6tD41zcicyFWOpFZAQ9w2SpeUdMUpKFgaTrzAkzfJuDUCiS5NIBHLpO19C4/iR", - "+YvbOWw6euW9TwSGLtU7LmrCh4jJ2lM+53sZ79qltM+ENY1yaY44RnHDATkKpCNuszWdZBnSLejXFspl", - "ChGR0zEXOSl4tY3kSTiU5yjPHWDKNQtpsxfTNSPjjVRSm645QwmZzYubCzBDq3XuQfJ6k89QISBV5x2O", - "djuQlZH/NVoX7Stwc3EDbu4avXYTdFtPoNG7bnbV5zEZE+e2fdW4qGtDjTZa9bOeUXu6nKG3zjHU7f7T", - "ogovLtp2B9qi1pkWl7lGsXtktY22t7wQ7v20isakNzDP7qrHUziquPdnFee83ym5M0TQIKeNnNfX29nV", - "6pZbj0V6+7hovd0NJ4XmVb9pNC/M2WPttjgmb88z1taa7Dx/W1yw7sSGnm7dHeF7SOpn3CnUnlqvfFKp", - "35Wqurhj/dLtk/5gngyOHvGNcV8bjEm3MR3lS/P7xrXeH/Kn0kkPNslx2y1cz91au0VzbdS6fyq8Os3r", - "mzrs5iedy5JnmOWmh2b8aDQck8Xtwwg1e0vvuXd83X+k1zfdxbx/aywnZuHxrDb3nvNdMc1pV5fFJfTy", - "S4fXvZPLjotm8+ubwdIek9WrmK6eDUbvMTpfuYtnc367EIT0azlz2PJynfsRe8pXik7rblRtapNqeaZd", - "no/Ojf7MJrOL3JjkjbtyfQAr+fJlaTnNz8QEleZd7eaR3lx73cY9vxzO8/m7i6f66gZ5q6NaVbvLPbWs", - "fnVWGt53p2NyjNrP5gr3r/MLu/B0cTboap69mPGT+pFnz8wCHU3KvPTmPM9v8tULOlo+lItT2K08DI+u", - "rGeExqR2nH+k99ZEK3Td4dHUeKZTzlriuXYzuXs+epqf1wYu0x/qbHo56cyKHXfQrS9H1pLf1nnDuiiM", - "Sb7nLYsPsN/Im8V25Ubr652c9jql+ZqmsWnj0cPLB4Yr2DvpP7q111HOGL5dOVxvm6SWe33ujgmu3Xq2", - "4VWr3qv1kFuI4kQQLMwBf51ay743fborP0/K1kyc16zuXe7xsVouvlq9SndRH9Rv640xEWfnF88Pg7nm", - "tMzuWb/QHdZrz879bFLqWL1Rv9B7bKzgQ8HSiF0P32uXnTl07qd6szIfE83RjvBt57rR6Dea9Xr5HLda", - "6PLYYdb5ZdW757e9fr+Yf6pozxZZPtXO647SoebFonbeXMzaY9JYtC/Ob2mnWefNRuOpWV+0mpdmq3le", - "rteb5ux20/vo6qmeqzaeXNNeDevPT5fWdNW1xiR3ZBy/3Rj388llMd96Lc3a1evzxlWe9B6PGncFx5sP", - "j15H3rD00GONklO68GzhdgetTrcnnErrbEwK7OLtsU5HhZV78tSu9epner/ZvF5N61NOH+5q1ac7r3mU", - "m5ApG6FBsTe4bhqrm2b1+OGkVsHX92PiVIZHE357tqg2iz1m6/V+uX/m0dVzYYjFBXwud2979+Jo1IKF", - "MuZPw4vm9I1Wb55q96XO9aySHxPz9cGsFa9yE6fYehtWR7XSQ+tsUrDn03Lbni/N9msXmYXC2+PT0mFP", - "w+dOp2nM34wj+2p47C3NyzGZLnOd/Mp+Lvbw5IIdX9Trq+uTuwdWfx4uhv18S5uOaotWkyxnwzNv9eo8", - "LO7nV41Hr9W+r12j0tOY9PFdwehc1bhePXP5+bLSP3rUSZ/cDo8u2XR00z0rOQ/MruukNbL0p/va9Hnm", - "PlhnK17KnZyg6zGxZnnWI6v89Goxg56Rw3e1a+34cd6fTXuDfses3J3cd1cd7+FBvC0eybR/VXkYnDde", - "u2X+TJ1+f0wMMRldFo4qq8ngIVcvzRsTuBw8FEX17u1qqr2h2fC5hWHv6qSXu9Q6zfagcHteO64Vz/S6", - "3To/0cdkVjRv8dPwtg5hJ9/p1N8u54PZoNPrmd3i0+0Tvry6XxVFqbM6NziDTmUxbD5cG9YNaq96jdFz", - "Z0zmzL2ybybI4KOTSnVkFBtXbc98e2bNyv3ybNidPZsDq3B/MR+2b0lz9Ta7XR237oqvNy5+qJxIG2Xd", - "tB+fWZdq3VK3NzzJ4bfO7Whgi2m//uuY/HpjjKpjolaX1tXZR0tPYsBFZahfOLeTl0oHCWhjMktevx3M", - "GGU8YdsV9vunXC1/9b9nSsWxl88Xj6UH8es6CLRvMfeR2MEeYpuINQ3yc1ZDRFCu8P8z8Fd+rWW4YAg6", - "EcxQ/ntc9t8o+uQW9Xp4AC3R9H9iJgwTM/QYgF8joNz3jc8AIJduBQdYJSU2AXNVejAmv7jYRTYm6Gti", - "GUIsZBrWJdFP1ngwizv7D+CdIYGYgwniYGGhYDPjZyy2DsQot8h3ItWoVDgmyV1KKAiLueHDnfz7jg+u", - "CTz3U8eBE7d9KhNpDImM/BSZThdyvqAssVRBepYviS5q3EM9QEQw4di0dk6hJlfDpVOUmZAEVTG7gZBy", - "vlQsvx8FiZMcnZGsnN8I5XsJ39kGbBGW3mX6Fg0RDkZGn7S/ipfykdXBB1l2IqbpvX12T6Hu6xLLgO/F", - "ET/XuK/LO1WN37+lY5lYzMMDXAxB2y+KUgd/wMQTID4+qYNQaSUSgBpjksC2LFBwHQRJkK2Ctg0SGgJ/", - "0viYQIYAtDkNtD6GF67bBvnwOaaqKs8/3yoJHhPm2cgv+mLIoAylwQIBC87XOXslCEClm+XoJgjAhZ/9", - "hEIdU+PkixgTl3KOJ7bq5uClyjw7UGgWcChDIOAyENRUtkoay7XYvRfZigTsP3Ouajtsfbg0HthjN1X0", - "CVk8sEdyEamqX/l84H4d+j8kdeN3DHI371WcB3GIcHa+7czjJ1MAzCPkvTh/lJykQH+Wl9ZB+DCevwmo", - "J0LkSSXiKhO5nW3a2G71MfGCg9ipgN1Fj3Mrg/RipVI4AfV6vd4sXb3BZsF+PmsXrkatinzXvmIX3Rbr", - "P+Gjfv9u4V3CQb3jDHq0/TYwiq9nRf2s8pZvjJa542USEfEQv8cR2x/xfid1qBYxzWNYrIZSKHwGNRBk", - "PuMm6td5uGB1HkbhVRlq/fTbraHKpdq/MAMTg8bdsmFQCSBo4Eupihw/BeInqrl0TWysoeAoZHBHR92F", - "moVAUSUj1HK79nwXi0UWqs/K3Qz68lyv3WxdDVuZYjaftYRjqxnEQrHsethQ6IN0GwOq5AVAF0cibaep", - "YsovvCPyw2mqlM1nCypkLSzFppxmU4J47nesf1dylVSUdYGEX/CitEWVZ4FACQBlQGqljURYb+0fRVBh", - "akmZquDBPDhKFfH9KFP5/E3hkaptlw6gUj+kIz0bLe5r6z4p0UOZ6a0bX35LPtUcHGD2iRcUmKosTN2j", - "IvmwuUYlOAQSSpzvZ20uVfnhZye/qSPE6sysmoxiPh/WhweJAOi6dnAeIjcNaiM3BH1okSNcUuK8zZko", - "T6SIlH8g6iDpG0faJv66H0gGwLqPuvDXo657wgKCzpDaXmCfEB976a/HfkegJyzK8Ju/pXERk7IB1rLt", - "U1L+OyiZEbogO1NQ+Ttm/46gpYs0aSdUIQGgmuYxqWlRE660ODTev32TOsI9x4FsFZQURo2QMl5reVJw", - "clrkRiOadCilqY7QAAgIWoRd08ClcuhYOccaJTyoPVX143PEYGjclb0PijHV1U1+MSBmQEeyS1BYGDNc", - "N5SL8OS8b2QQF+EZ/B+j8dunR79vL5/SmH2P2ZvCj8be1pOmPvgILMjl/DGB9H+b0WGbc6U/Lc9Py3Og", - "5QmMRpKl+VHO0yf8pZCHexylrcPNB7lKa8D/x5ylLU4lSNA2X346TD/N1n+pw/Su/fI3glGvKcF/iV7/", - "c5A9iRir/yAr8hf4XrsXK/3d3lfSNUwJIqWON6HFpqJ+guQ6FJwqT7ZrAi1FzrUh3qEn4Y7Rw6xX+Uch", - "SNLN71urtmTL1lmyDxTADooh/8gqbmCCuRVZxMGHazgWm6U7rXYUKiruIAEBJr4MY0oAnFBPhBfYebb4", - "aJlXtZw/F/m9i3xwbVOiakgRWB/58y99XG8QMQGEqrwq1jwbsuCME/hFWNQzrSCl0RleX33N/s8p0oU6", - "z2aGGehQypPUaOsKqg91ad3yAHUaqPtQubpEc32PhCRG7cEDc0aiN29mgTrEtm6sUaVYPDy9F0yfjgys", - "7lkVIBqODe6E8MsPIAnviMiE4LKVD1Rxc7XXT33cq48bZr2jlFvTHVPM/01d21aPA5QuUmX8sc4FDX2V", - "i+mZf9oWLaEmthai9XXEOnIR0fnmohala+vQvzoD+pFmhHT+VIz9irG+Pe4dvQin8jN68XOT+nOT+p+2", - "SY3ZpiR7p4BHfYqYidlc8REzLkkj2zTJqQM875U4RNqpEz5/qepvxpAk7f6dlNQAATN+qtm/R818Qf/v", - "UzK4FiBo22Bd/xRK00bN9ke0IfGLHoi2vmPdp2xzG8lkBdTSmayoh8ePUND8T636pb95DX93KtUHEH33", - "U4t/avFntBjFJUhq7rrI5/0V8jpokiz328QG4JQ+y5215EGwZ/5v9C0+HM73dR10kiXqB1ejUN3T/Pt8", - "1kevt8u4oIuzEg+3cPD/N4Au9q9GzajoAWKZ8F6m3LyoPI6d4jIBTUzMjxBwAU30J9EoJpLw6pY1mn1w", - "vn3//wEAAP//qIL3tG9rAAA=", + "oM/g6Mu/pPBpvbT+uByi8vAl/JfdHBHkGiI6JCIzYRDrmVK+VCmU9joKEXDpfSnJrVhvvDCJaRYWSBMe", + "2xnOsnb8clx+f3n3Xx8QMh2tXKQCnX6KYV+f6+FItlIj3t7Q/oAtmb/Iv1D3oAD/tosVq62Ksm6LKzuk", + "fwtn4T2JQuF24eAA8Nrr/XQAPAgdr1lxGIDtFGpy3HlnmJ+K6aZTavvu//Qp83+HxTNB4DcmixEJi6CC", + "C4kGLnjGghlmeTh4ivzk0F0/vvnE+HU0wUukmyizTm8FT2GMKHiBCRfQttULU3P9f0MAplS4tUVQf7c6", + "YCo28P2HDXj5vNuYwcUa3Jy70ttLZEk3SN4kB1c3Cn6OdMpgpik9uUwD8nd8TxtBvtOzmC/m8yf5ajaf", + "6E8hNveD7pseods4o1OcNRTiwIZlKTPVa8ubbGXDGU6O3/LZrhUtF9MJiYk5YjyWhivtL70LyN+gCoqA", + "NxA3XEmyt+vMdcLCIVffIJ1CVJY1tk8lvjj4Ld8D/54dUTp0CHeSYhbh3n0b5AyT5FBCWMsdZ3y4X45/", + "EVRAO+nTDhcU0vS6CNyvvfY7p9/dyqdTwcoRr1+FDJGErXATErnZQVjFEmGQzAa/BKw7Bfnicb48Kerw", + "GJ1UyhO9VJ7UJrUirJUqqAKrVb04Oc4bBvyali4YBBMGiWZlbDxDgK2zPht4zEJ2rpbzV9acNCFfd3Yo", + "8RbJSmnEs8v7u71bFhrn5E5AKsZSKyAh7hskS8s7YpSUIwwmX2FImuXd1H+iS5NIBHLpO19C4/iR+Yvb", + "OWw6euW9TwSGLtU7nmnCh4jJ2lM153sZ79qltM+ENY1yaY44RnHDATkKpCNuszWdZBnSLeiXFMplChGR", + "0zEXOSl4tY3kSTiU5yjPHWDKNQtpsxfTNSPjjRRQm645QwkJzYubCzBDq3XKQfJ6k8ZQkR9V3h2Odjt+", + "lZH/NVoX7Stwc3EDbu4avXYTdFtPoNG7bnbV5zEZE+e2fdW4qGtDjTZa9bOeUXu6nKG3zjHU7f7Togov", + "Ltp2B9qi1pkWl7lGsXtktY22t7wQ7v20isakNzDP7qrHUziquPdnFee83ym5M0TQIKeNnNfX29nV6pZb", + "j0V6+7hovd0NJ4XmVb9pNC/M2WPttjgmb88z1taa7Dx/W1yw7sSGnm7dHeF7SOpn3CnUnlqvfFKp35Wq", + "urhj/dLtk/5gngyOHvGNcV8bjEm3MR3lS/P7xrXeH/Kn0kkPNslx2y1cz91au0VzbdS6fyq8Os3rmzrs", + "5iedy5JnmOWmh2b8aDQck8Xtwwg1e0vvuXd83X+k1zfdxbx/aywnZuHxrDb3nvNdMc1pV5fFJfTyS4fX", + "vZPLjotm8+ubwdIek9WrmK6eDUbvMTpfuYtnc367EIT0azlz2PJynfsRe8pXik7rblRtapNqeaZdno/O", + "jf7MJrOL3JjkjbtyfQAr+fJlaTnNz8QEleZd7eaR3lx73cY9vxzO8/m7i6f66gZ5q6NaVbvLPbWsfnVW", + "Gt53p2NyjNrP5gr3r/MLu/B0cTboap69mPGT+pFnz8wCHU3KvPTmPM9v8tULOlo+lItT2K08DI+urGeE", + "xqR2nH+k99ZEK3Td4dHUeKZTzlriuXYzuXs+epqf1wYu0x/qbHo56cyKHXfQrS9H1pLf1nnDuiiMSb7n", + "LYsPsN/Im8V25Ubr652c9jql+ZqmsWnj0cPLB4Yr2DvpP7q111HOGL5dOVxvm6SWe33ujgmu3Xq24VWr", + "3qv1kFuI4kQQLMwBf51ay743fborP0/K1kyc16zuXe7xsVouvlq9SndRH9Rv640xEWfnF88Pg7nmtMzu", + "Wb/QHdZrz879bFLqWL1Rv9B7bKzgQ8HSiF0P32uXnTl07qd6szIfE83RjvBt57rR6Dea9Xr5HLda6PLY", + "Ydb5ZdW757e9fr+Yf6pozxZZPtXO647SoebFonbeXMzaY9JYtC/Ob2mnWefNRuOpWV+0mpdmq3lerteb", + "5ux20/vo6qmeqzaeXNNeDevPT5fWdNW1xiR3ZBy/3Rj388llMd96Lc3a1evzxlWe9B6PGncFx5sPj15H", + "3rD00GONklO68GzhdgetTrcnnErrbEwK7OLtsU5HhZV78tSu9epner/ZvF5N61NOH+5q1ac7r3mUm5Ap", + "G6FBsTe4bhqrm2b1+OGkVsHX92PiVIZHE357tqg2iz1m6/V+uX/m0dVzYYjFBXwud2979+Jo1IKFMuZP", + "w4vm9I1Wb55q96XO9aySHxPz9cGsFa9yE6fYehtWR7XSQ+tsUrDn03Lbni/N9msXmYXC2+PT0mFPw+dO", + "p2nM34wj+2p47C3NyzGZLnOd/Mp+Lvbw5IIdX9Trq+uTuwdWfx4uhv18S5uOaotWkyxnwzNv9eo8LO7n", + "V41Hr9W+r12j0tOY9PFdwehc1bhePXP5+bLSP3rUSZ/cDo8u2XR00z0rOQ/MruukNbL0p/va9HnmPlhn", + "K17KnZyg6zGxZnnWI6v89Goxg56Rw3e1a+34cd6fTXuDfses3J3cd1cd7+FBvC0eybR/VXkYnDdeu2X+", + "TJ1+f0wMMRldFo4qq8ngIVcvzRsTuBw8FEX17u1qqr2h2fC5hWHv6qSXu9Q6zfagcHteO64Vz/S63To/", + "0cdkVjRv8dPwtg5hJ9/p1N8u54PZoNPrmd3i0+0Tvry6XxVFqbM6NziDTmUxbD5cG9YNaq96jdFzZ0zm", + "zL2ybybI4KOTSnVkFBtXbc98e2bNyv3ybNidPZsDq3B/MR+2b0lz9Ta7XR237oqvNy5+qJxIG2XdtB+f", + "WZdq3VK3NzzJ4bfO7Whgi2m//uuY/HpjjKpjolaX1tXZR0tPYsBFJaZfOLeTl0oHCWhjMktevx3MGGU8", + "YdsV9vunXC1/9b9nSsWxl88Xj6UH8es6CLRvMfeR2MEeYpuINQ3yc1ZDRFCu8P8z8Fd+rWW4YAg6EcxQ", + "/ntc9t8o+uQW9Xp4AC3RrH9iAgwTM/QYgF8aoNz3jc8AIJduBQdY5SI2cXJVcTAmv7jYRTYm6Gti9UEs", + "UhqWI9FPlnYwizv7z92dIYGYgwniYGGhYDPjJyq2zsEot8h3ItWoVDgmyV1KqAOLueHDnbT7jg+uCTz3", + "M8aBE7d9GBNpDImM/BSZThdyvqAssUJBepYviS5q3EM9QEQw4di0dg6fJhfBpVOUmZAExTC7gZByvlQs", + "vx8FiZMcnZGsnN8I5XsJ39kGbBGW3mX6Fg0RDkZGn7S/ilfwkdXB51d2IqbpvX12D5/u6xJLfO/FET/O", + "uK/LO8WM37+lYwlYzMNzWwxB26+FUud9wMQTID4+qYNQaSUSgBpjksC2LFBwHQRJkKSCtg0SGgJ/0viY", + "QIYAtDkNtD6GF67bBmnwOaaqGM8/1ioJHhPm2civ9WLIoAylwQIBC87XqXolCEBlmeXoJgjAhZ/0hEKd", + "TuPkixgTl3KOJ7bq5uClSjg7UGgWcChDIOAyENRUtkoay7XYvRfZigTsP3Ocajtsfbg0HthjN1X0CVk8", + "sEdy7agqW/l84H4d+j8kdeN3DHI37xWaB3GIcHa+7czjJ1MAzCPkvTh/lJykQH+Wl9ZB+DCevwmoJ0Lk", + "SZXhKgG5nW3a2G71MfFeg9hhgN1Fj3Mrg/RipVI4AfV6vd4sXb3BZsF+PmsXrkatinzXvmIX3RbrP+Gj", + "fv9u4V3CQb3jDHq0/TYwiq9nRf2s8pZvjJa542USEfEQv8cR2x/xfid1qBYxzWNYrIZSKHwGNRBkPuMm", + "6td5uGB1HkbhDRlq/fTbraHKpdq/JwMTg8bdsmFQACBo4EupQhw/BeLnp7l0TWysoeAEZHA1R92FmoVA", + "USUj1HK79nwXi0UWqs/K3Qz68lyv3WxdDVuZYjaftYRjqxnEQrHsethQ6IN0GwOq0gVAF0cibaepYsqv", + "tyPyw2mqlM1nCypkLSzFppxmU4J47nesf1dylVSLdYGEX+eitEVVZYFACQBlQGqljURYZu2fQFBhakmZ", + "KtzBPDhBFfH9KFNp/E29kSpplw6gUj+kIz0brelr6z4p0bOY6a2LXn5LPswcnFv2iRcUmKoaTF2fIvmw", + "uT0lOPsRSpzvZ23uUvnhRya/qZPD6qismoxiPh+WhQeJAOi6dnAMIjcNSiI3BH1okSNcUuK8zZkoT6SI", + "lH8g6iDpG0faJv66H0gGwLqPuvDXo657wgKCzpDaXmCfEB976a/HfkegJyzK8Ju/pXERk7IB1rLtU1L+", + "OyiZEbogO1NQ+Ttm/46gpYs0aSdUIQGgmuYxqWlRE660ODTev32TOsI9x4FsFVQSRo2QMl5reVJwclrk", + "IiOadBalqU7OAAgIWoRd08ClcuhYOccaJTwoOVVl43PEYGjclb0PajDVjU1+DSBmQEeyS1BPGDNcN5SL", + "8MC8b2QQF+HR+x+j8duHRr9vL5/SmH2P2ZvCj8be1pOmPvgILMjl/DGB9H+b0WGb46Q/Lc9Py3Og5QmM", + "RpKl+VHO0yf8pZCHexylrTPNB7lKa8D/x5ylLU4lSNA2X346TD/N1n+pw/Su/fI3glGvKcF/id76c5A9", + "iRir/yAr8hf4Xrv3Kf3d3lfS7UsJIqVONaHFppB+guQ6FBwmT7ZrAi1FzrUh3qEn4WrRw6xX+UchSNLN", + "71urtmTL1hGyDxTADooh/8gqbmCCuRVZxMGHazgWm6U7rXYUKiruIAEBJr4MY0oAnFBPhPfWebb4aJlX", + "tZw/F/m9i3xwW1OiakgRWJ/08+96XG8QMQGEqrwq1jwbsuBoE/hFWNQzrSCl0RleX33N/s8p0oU6xmaG", + "GehQypPUaOvmqQ91ad3yAHUaqGtQubo7c319hCRG7cEDc0aiF25mgTq7tm6sUaVYPDy0F0yfjgysrlcV", + "IBqODa6C8MsPIAmvhsiE4LKVD1Rxc6PXT33cq48bZr2jlFvTHVPM/01d21aPA5QuUmX8sc4FDX2Vi+mZ", + "f8gWLaEmthai9S3EOnIR0fnmfhala+vQvzr6+ZFmhHT+VIz9irG+NO4dvQin8jN68XOT+nOT+p+2SY3Z", + "piR7p4BHfYqYidnc7BEzLkkj2zTJqQM875U4RNqpEz5/qepvxpAk7f5VlNQAATN+qtm/R818Qf/vUzK4", + "FiBo22Bd/xRK00bN9ke0IfGLHoi2vlrdp2xzCclkBdTSmayoh8ePUND8T636pb95DX93KtUHEH33U4t/", + "avFntBjFJUhq7rrI5/0V8jpokiz328QG4JQ+y5215EGwZ/5v9C0+HM73dR10kiXqBzeiUN3T/Gt81kev", + "t8u4oIuzEg+3cPC/NYAu9m9EzajoAWKZ8Dqm3LyoPI6d4jIBTUzMjxBwAU30J9EoJpLwxpY1mn1wvn3/", + "/wEAAP//PejbA2ZrAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/cloudapi/v2/openapi.v2.yml b/internal/cloudapi/v2/openapi.v2.yml index cf6967052f..1a0674446e 100644 --- a/internal/cloudapi/v2/openapi.v2.yml +++ b/internal/cloudapi/v2/openapi.v2.yml @@ -862,7 +862,6 @@ components: type: object required: - region - - bucket properties: region: type: string diff --git a/test/cases/api/gcp.sh b/test/cases/api/gcp.sh index d88877c8ad..1d9f8c454c 100644 --- a/test/cases/api/gcp.sh +++ b/test/cases/api/gcp.sh @@ -78,7 +78,6 @@ function createReqFile() { "image_type": "${IMAGE_TYPE}", "repositories": $(jq ".\"$ARCH\"" /usr/share/tests/osbuild-composer/repositories/"$DISTRO".json), "upload_options": { - "bucket": "${GCP_BUCKET}", "region": "${GCP_REGION}", "image_name": "${GCP_IMAGE_NAME}", "share_with_accounts": ["${GCP_API_TEST_SHARE_ACCOUNT}"] diff --git a/tools/provision.sh b/tools/provision.sh index dc2c0cd120..93f8cccbde 100755 --- a/tools/provision.sh +++ b/tools/provision.sh @@ -87,7 +87,12 @@ EOF # it into /tmp and as a result, the worker would not see it due to using PrivateTmp=true. GCP_CREDS_WORKER_PATH="/etc/osbuild-worker/gcp-credentials.json" sudo cp "$GOOGLE_APPLICATION_CREDENTIALS" "$GCP_CREDS_WORKER_PATH" - echo -e "\n[gcp]\ncredentials = \"$GCP_CREDS_WORKER_PATH\"\n" | sudo tee -a /etc/osbuild-worker/osbuild-worker.toml + sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF + +[gcp] +credentials = "$GCP_CREDS_WORKER_PATH" +bucket = "$GCP_BUCKET" +EOF fi # if Azure credentials are defined in the env, create the credentials file From d616f6b68d62bda8091a786c32b9c77fac1e9599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Mon, 3 Oct 2022 14:31:40 +0200 Subject: [PATCH 07/10] worker/gcp: return error if job doesn't specify object key The object key is required in order to upload the image to GCP. Return an error if it is not set. --- cmd/osbuild-worker/jobimpl-osbuild.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index adb920a186..5747c97bbd 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -580,6 +580,11 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { break } + if targetOptions.Object == "" { + targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, "No GCP object key provided", nil) + break + } + bucket := targetOptions.Bucket if bucket == "" { bucket = impl.GCPConfig.Bucket From e000bf1e002dec2ef25f86c38671436933c3c01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Tue, 27 Sep 2022 09:14:08 +0200 Subject: [PATCH 08/10] weldr/upload/gcp: make `Object` optional Previously, it was expected from the user to provide the Object name when uploading image to GCP. The object name does not matter much, because the object is deleted once image import finishes. Make the specification of the object name optional and generate it if not provided. Adjust the GCP Weldr test case to not provide the Object name when uploading the image. The user can still provide the Object name if needed. --- internal/weldr/upload.go | 10 +++++++--- test/cases/gcp.sh | 1 - 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/weldr/upload.go b/internal/weldr/upload.go index 04ab1626a6..2e905085d1 100644 --- a/internal/weldr/upload.go +++ b/internal/weldr/upload.go @@ -65,7 +65,7 @@ func (azureUploadSettings) isUploadSettings() {} type gcpUploadSettings struct { Region string `json:"region"` Bucket string `json:"bucket"` - Object string `json:"object"` + Object string `json:"object,omitempty"` // base64 encoded GCP credentials JSON file Credentials string `json:"credentials,omitempty"` @@ -300,9 +300,13 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target. } } - // The uploaded image object name must have 'tar.gz' suffix to be imported + // Providing the Object name is optional. If it is provided, we must + // ensure that it has a '.tar.gz' suffix to be successfully imported. + // If it is not provided, we will generate a random name. objectName := options.Object - if !strings.HasSuffix(objectName, ".tar.gz") { + if objectName == "" { + objectName = fmt.Sprintf("composer-api-%s.tar.gz", uuid.New().String()) + } else if !strings.HasSuffix(objectName, ".tar.gz") { objectName = objectName + ".tar.gz" logrus.Infof("[GCP] object name must end with '.tar.gz', using %q as the object name", objectName) } diff --git a/test/cases/gcp.sh b/test/cases/gcp.sh index ecf03d55f5..457aace8ea 100755 --- a/test/cases/gcp.sh +++ b/test/cases/gcp.sh @@ -201,7 +201,6 @@ provider = "gcp" [settings] bucket = "${GCP_BUCKET}" region = "${GCP_REGION}" -object = "${GCP_IMAGE_NAME}" credentials = "$(base64 -w 0 "${GOOGLE_APPLICATION_CREDENTIALS}")" EOF From a7ab90cfd48e9f82c7cd95ae768ff4dafc1d2d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Mon, 3 Oct 2022 16:31:48 +0200 Subject: [PATCH 09/10] templates/packer: set the GCP bucket in the worker configuration Similar to AWS, set the GCP bucket in the worker configuration. --- .../worker-initialization-scripts/get_gcp_creds.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_gcp_creds.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_gcp_creds.sh index 1b8f816a46..d99a62cddf 100755 --- a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_gcp_creds.sh +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_gcp_creds.sh @@ -4,6 +4,15 @@ source /tmp/cloud_init_vars echo "Deploy GCP credentials." +echo "Write the bucket." +# Always create the header and write the bucket, it's slightly ugly but it will work +# The bucket is always set, becuase the instance can potentially authenticate to GCP +# with a service account connected to it, without any explicit credentials. +sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF +[gcp] +bucket = "${WORKER_CONFIG_GCP_BUCKET:-}" +EOF + if [[ -z "$GCP_SERVICE_ACCOUNT_IMAGE_BUILDER_ARN" ]]; then echo "GCP_SERVICE_ACCOUNT_IMAGE_BUILDER_ARN not defined, skipping." exit 0 @@ -16,6 +25,5 @@ fi sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF -[gcp] credentials = "/etc/osbuild-worker/gcp_credentials.json" EOF From a80273ceb1661e4b18a8bc77c77ccc0975540f93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Mon, 3 Oct 2022 16:41:17 +0200 Subject: [PATCH 10/10] templates/packer: add comment to `get_aws_creds.sh` Add a comment explaining why it is important to set the AWS bucket in the worker configuration, even if the `AWS_ACCOUNT_IMAGE_BUILDER_ARN` is empty. --- .../common/files/worker-initialization-scripts/get_aws_creds.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_aws_creds.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_aws_creds.sh index 22f0b806ac..b9571e1af0 100755 --- a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_aws_creds.sh +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/get_aws_creds.sh @@ -7,6 +7,8 @@ echo "Deploy AWS credentials." echo "Write the bucket." # Always create the header and write the bucket, it's slightly ugly but it will work +# The bucket is always set, becuase the instance can potentially authenticate to AWS +# with its instance profile, without any explicit credentials. sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF [aws] bucket = "${WORKER_CONFIG_AWS_BUCKET:-}"