From c0dd8d429b82e9ec3399d1b6c0c6a54dd898f708 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 11 Jun 2020 17:19:00 -0700 Subject: [PATCH 01/28] variants: Remove reference to top-level list --- variants/README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/variants/README.md b/variants/README.md index 168927d96fb..79c213c8a54 100644 --- a/variants/README.md +++ b/variants/README.md @@ -130,20 +130,6 @@ We use the same Rust code for all variants. ### Next Steps -After the above files are in place, the variant must be added to the `variants` workspace. - -Open `variants/Cargo.toml` in your editor and update the `members` list. -``` -[workspace] -members = [ - ... - "my-variant", - ... -] -``` - -Next, run `cargo generate-lockfile` to refresh `Cargo.lock`. - To build your variant, run the following command in the top-level Bottlerocket directory. ``` cargo make -e BUILDSYS_VARIANT=my-variant From 6295501a94ec689a22a74e6e0383c8e9f05f53b1 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 11 Jun 2020 17:22:29 -0700 Subject: [PATCH 02/28] variants: create aws-ecs-1 variant --- variants/README.md | 5 +++++ variants/aws-ecs-1/Cargo.lock | 5 +++++ variants/aws-ecs-1/Cargo.toml | 20 ++++++++++++++++++++ variants/aws-ecs-1/build.rs | 9 +++++++++ variants/aws-ecs-1/lib.rs | 1 + 5 files changed, 40 insertions(+) create mode 100644 variants/aws-ecs-1/Cargo.lock create mode 100644 variants/aws-ecs-1/Cargo.toml create mode 100644 variants/aws-ecs-1/build.rs create mode 100644 variants/aws-ecs-1/lib.rs diff --git a/variants/README.md b/variants/README.md index 79c213c8a54..9df45887f51 100644 --- a/variants/README.md +++ b/variants/README.md @@ -45,6 +45,11 @@ It supports self-hosted clusters and clusters managed by [EKS](https://aws.amazo This variant is compatible with Kubernetes 1.16, 1.17, and 1.18 clusters. +### aws-ecs-1: Amazon ECS container instance + +The [aws-ecs-1](aws-ecs-1/Cargo.toml) variant includes the packages needed to run an [Amazon ECS](https://ecs.aws) +container instance in AWS. + ### aws-dev: Development build The [aws-dev](aws-dev/Cargo.toml) variant has useful packages for local development of the OS. diff --git a/variants/aws-ecs-1/Cargo.lock b/variants/aws-ecs-1/Cargo.lock new file mode 100644 index 00000000000..f70b2ee3691 --- /dev/null +++ b/variants/aws-ecs-1/Cargo.lock @@ -0,0 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aws-ecs-1" +version = "0.1.0" diff --git a/variants/aws-ecs-1/Cargo.toml b/variants/aws-ecs-1/Cargo.toml new file mode 100644 index 00000000000..b8d353d33df --- /dev/null +++ b/variants/aws-ecs-1/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "aws-ecs-1" +version = "0.1.0" +edition = "2018" +publish = false +build = "build.rs" + +[package.metadata.build-variant] +included-packages = [ +# core + "release", +# docker + "docker-cli", + "docker-engine", + "docker-init", + "docker-proxy", +] + +[lib] +path = "lib.rs" diff --git a/variants/aws-ecs-1/build.rs b/variants/aws-ecs-1/build.rs new file mode 100644 index 00000000000..d6a90e4df44 --- /dev/null +++ b/variants/aws-ecs-1/build.rs @@ -0,0 +1,9 @@ +use std::process::{exit, Command}; + +fn main() -> Result<(), std::io::Error> { + let ret = Command::new("buildsys").arg("build-variant").status()?; + if !ret.success() { + exit(1); + } + Ok(()) +} diff --git a/variants/aws-ecs-1/lib.rs b/variants/aws-ecs-1/lib.rs new file mode 100644 index 00000000000..d799fb2d44c --- /dev/null +++ b/variants/aws-ecs-1/lib.rs @@ -0,0 +1 @@ +// not used From 062d3ad0d2ef9998b109281c065115b910e3d1f8 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 15 Jul 2020 14:57:30 -0700 Subject: [PATCH 03/28] containerd: add config template for aws-ecs-1 --- .../containerd/containerd-config-toml_aws-ecs-1 | 14 ++++++++++++++ packages/containerd/containerd.spec | 7 ++++--- 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 packages/containerd/containerd-config-toml_aws-ecs-1 diff --git a/packages/containerd/containerd-config-toml_aws-ecs-1 b/packages/containerd/containerd-config-toml_aws-ecs-1 new file mode 100644 index 00000000000..dbd68745aed --- /dev/null +++ b/packages/containerd/containerd-config-toml_aws-ecs-1 @@ -0,0 +1,14 @@ +version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" +disabled_plugins = [ + "io.containerd.internal.v1.opt", + "io.containerd.snapshotter.v1.aufs", + "io.containerd.snapshotter.v1.devmapper", + "io.containerd.snapshotter.v1.native", + "io.containerd.snapshotter.v1.zfs", + "io.containerd.grpc.v1.cri", +] + +[grpc] +address = "/run/containerd/containerd.sock" diff --git a/packages/containerd/containerd.spec b/packages/containerd/containerd.spec index 3300bc4804d..ce86dcfa0e3 100644 --- a/packages/containerd/containerd.spec +++ b/packages/containerd/containerd.spec @@ -17,7 +17,8 @@ Source0: https://%{goimport}/archive/v%{gover}/%{gorepo}-%{gover}.tar.gz Source1: containerd.service Source2: containerd-config-toml_aws-k8s Source3: containerd-config-toml_aws-dev -Source4: containerd-tmpfiles.conf +Source4: containerd-config-toml_aws-ecs-1 +Source5: containerd-tmpfiles.conf Source1000: clarify.toml # Upstream patch; can drop when we move to v1.4.0. @@ -77,10 +78,10 @@ install -p -m 0644 %{S:1} %{buildroot}%{_cross_unitdir}/containerd.service install -d %{buildroot}%{_cross_templatedir} install -d %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir}/containerd -install -p -m 0644 %{S:2} %{S:3} %{buildroot}%{_cross_templatedir} +install -p -m 0644 %{S:2} %{S:3} %{S:4} %{buildroot}%{_cross_templatedir} install -d %{buildroot}%{_cross_tmpfilesdir} -install -p -m 0644 %{S:4} %{buildroot}%{_cross_tmpfilesdir}/containerd.conf +install -p -m 0644 %{S:5} %{buildroot}%{_cross_tmpfilesdir}/containerd.conf %cross_scan_attribution --clarify %{S:1000} go-vendor vendor From 13e6b1b934d6b13018a7b7761646539dfe264c15 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 11 Jun 2020 20:46:10 -0700 Subject: [PATCH 04/28] models: add initial model for aws-ecs-1 --- sources/models/README.md | 5 +++++ sources/models/src/aws-ecs-1/mod.rs | 18 ++++++++++++++++++ .../src/aws-ecs-1/override-defaults.toml | 3 +++ sources/models/src/lib.rs | 11 +++++++++++ 4 files changed, 37 insertions(+) create mode 100644 sources/models/src/aws-ecs-1/mod.rs create mode 100644 sources/models/src/aws-ecs-1/override-defaults.toml diff --git a/sources/models/README.md b/sources/models/README.md index 8a838f93071..a729e466254 100644 --- a/sources/models/README.md +++ b/sources/models/README.md @@ -35,6 +35,11 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and * [Model](src/aws-k8s-1.17/mod.rs) * [Overridden defaults](src/aws-k8s-1.17/override-defaults.toml) +### aws-ecs-1: Amazon ECS + +* [Model](src/aws-ecs-1/mod.rs) +* [Overridden defaults](src/aws-ecs-1/override-defaults.toml) + ### aws-dev: Development build * [Model](src/aws-dev/mod.rs) diff --git a/sources/models/src/aws-ecs-1/mod.rs b/sources/models/src/aws-ecs-1/mod.rs new file mode 100644 index 00000000000..9931426d2bc --- /dev/null +++ b/sources/models/src/aws-ecs-1/mod.rs @@ -0,0 +1,18 @@ +use model_derive::model; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use crate::modeled_types::Identifier; +use crate::{AwsSettings, ContainerImage, ECSSettings, NtpSettings, UpdatesSettings}; + +// Note: we have to use 'rename' here because the top-level Settings structure is the only one +// that uses its name in serialization; internal structures use the field name that points to it +#[model(rename = "settings", impl_default = true)] +struct Settings { + motd: String, + updates: UpdatesSettings, + host_containers: HashMap, + ntp: NtpSettings, + aws: AwsSettings, + ecs: ECSSettings, +} diff --git a/sources/models/src/aws-ecs-1/override-defaults.toml b/sources/models/src/aws-ecs-1/override-defaults.toml new file mode 100644 index 00000000000..e89c0ed270c --- /dev/null +++ b/sources/models/src/aws-ecs-1/override-defaults.toml @@ -0,0 +1,3 @@ +[configuration-files.containerd-config-toml] +# No override to path +template-path = "/usr/share/templates/containerd-config-toml_aws-ecs-1" \ No newline at end of file diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index 18789c63bdc..6d9b450378d 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -32,6 +32,11 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and * [Model](src/aws-k8s-1.17/mod.rs) * [Overridden defaults](src/aws-k8s-1.17/override-defaults.toml) +## aws-ecs-1: Amazon ECS + +* [Model](src/aws-ecs-1/mod.rs) +* [Overridden defaults](src/aws-ecs-1/override-defaults.toml) + ## aws-dev: Development build * [Model](src/aws-dev/mod.rs) @@ -99,6 +104,12 @@ struct KubernetesSettings { pod_infra_container_image: SingleLineString, } +// ECS settings. +#[model] +struct ECSSettings { + cluster: String, +} + // Update settings. Taken from userdata. The 'seed' setting is generated // by the "Bork" settings generator at runtime. #[model] From d9d430f2396632d9e0cb305f6f6e5861bb40b8bc Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 8 Jul 2020 22:02:02 -0700 Subject: [PATCH 05/28] ecs-agent: new package --- packages/Cargo.lock | 7 + packages/Cargo.toml | 1 + ...engine-move-default-image-exclusions.patch | 145 ++++++++++++++++++ ...erocket-default-filesystem-locations.patch | 82 ++++++++++ ...-version-values-settable-with-linker.patch | 30 ++++ packages/ecs-agent/Cargo.toml | 22 +++ packages/ecs-agent/README.md | 27 ++++ packages/ecs-agent/build.rs | 9 ++ packages/ecs-agent/ecs-agent.spec | 99 ++++++++++++ packages/ecs-agent/ecs-tmpfiles.conf | 2 + packages/ecs-agent/ecs.service | 34 ++++ packages/ecs-agent/pause-config.json | 1 + packages/ecs-agent/pause-image-VERSION | 1 + packages/ecs-agent/pause-manifest.json | 1 + packages/ecs-agent/pause-repositories | 1 + packages/ecs-agent/pkg.rs | 1 + 16 files changed, 463 insertions(+) create mode 100644 packages/ecs-agent/0001-engine-move-default-image-exclusions.patch create mode 100644 packages/ecs-agent/0002-bottlerocket-default-filesystem-locations.patch create mode 100644 packages/ecs-agent/0003-bottlerocket-version-values-settable-with-linker.patch create mode 100644 packages/ecs-agent/Cargo.toml create mode 100644 packages/ecs-agent/README.md create mode 100644 packages/ecs-agent/build.rs create mode 100644 packages/ecs-agent/ecs-agent.spec create mode 100644 packages/ecs-agent/ecs-tmpfiles.conf create mode 100644 packages/ecs-agent/ecs.service create mode 100644 packages/ecs-agent/pause-config.json create mode 100644 packages/ecs-agent/pause-image-VERSION create mode 100644 packages/ecs-agent/pause-manifest.json create mode 100644 packages/ecs-agent/pause-repositories create mode 100644 packages/ecs-agent/pkg.rs diff --git a/packages/Cargo.lock b/packages/Cargo.lock index 9312e6b48ad..3e6aedbdfed 100644 --- a/packages/Cargo.lock +++ b/packages/Cargo.lock @@ -126,6 +126,13 @@ dependencies = [ "glibc", ] +[[package]] +name = "ecs-agent" +version = "0.1.0" +dependencies = [ + "glibc", +] + [[package]] name = "filesystem" version = "0.1.0" diff --git a/packages/Cargo.toml b/packages/Cargo.toml index 954b6afb038..00ca82c534d 100644 --- a/packages/Cargo.toml +++ b/packages/Cargo.toml @@ -15,6 +15,7 @@ members = [ "docker-engine", "docker-init", "docker-proxy", + "ecs-agent", "filesystem", "findutils", "glibc", diff --git a/packages/ecs-agent/0001-engine-move-default-image-exclusions.patch b/packages/ecs-agent/0001-engine-move-default-image-exclusions.patch new file mode 100644 index 00000000000..0fc98ec4585 --- /dev/null +++ b/packages/ecs-agent/0001-engine-move-default-image-exclusions.patch @@ -0,0 +1,145 @@ +From c2a7cea2edeae59cdc6ca31696e36f7e922ae71f Mon Sep 17 00:00:00 2001 +From: Samuel Karp +Date: Wed, 8 Jul 2020 10:05:12 -0700 +Subject: [PATCH] engine: move default image exclusions + +This commit moves the logic adding default excluded images from the +cleanup list to the engine package, away from the config package, and in +doing so fixes two bugs: + +1. Prior to this change, any value for ImageCleanupExclusionList + provided by a config mechanism *other* than environmentConfig would + be ignored. This is because environmentConfig has the highest + precedence, the defaults were added to environmentConfig by + parseImageCleanupExclusionList, and config.Merge will only merge a + new value when the left config's field is its zero value. Since the + default excluded images were populated, environmentConfig's + ImageCleanupExclusionList field was never zero. +2. CachedImageNamePauseContainer hard-coded a name for the pause + container image that was used to populate the exclusion list, but the + actual name of the pause container is a value populated at link-time + into the DefaultPauseContainerImageName and DefaultPauseContainerTag + variables. If the value was set to anything other than what was + defined in CachedImageNamePauseContainer, the pause container image + would not be correctly excluded from image cleanup. + +Signed-off-by: Samuel Karp +--- + agent/config/config.go | 1 - + agent/config/config_test.go | 2 +- + agent/config/parse.go | 7 +------ + agent/engine/docker_image_manager.go | 15 ++++++++++++++- + agent/engine/docker_image_manager_test.go | 17 +++++++++++++++++ + 5 files changed, 33 insertions(+), 9 deletions(-) + +diff --git a/agent/config/config.go b/agent/config/config.go +index 1079684b..20f2f684 100644 +--- a/agent/config/config.go ++++ b/agent/config/config.go +@@ -132,7 +132,6 @@ const ( + DefaultTaskMetadataBurstRate = 60 + + //Known cached image names +- CachedImageNamePauseContainer = "amazon/amazon-ecs-pause:0.1.0" + CachedImageNameAgentContainer = "amazon/amazon-ecs-agent:latest" + + // DefaultNvidiaRuntime is the name of the runtime to pass Nvidia GPUs to containers +diff --git a/agent/config/config_test.go b/agent/config/config_test.go +index e4ae71ab..3ac2d9e2 100644 +--- a/agent/config/config_test.go ++++ b/agent/config/config_test.go +@@ -419,7 +419,7 @@ func TestValidForImagesCleanupExclusion(t *testing.T) { + defer setTestRegion()() + defer setTestEnv("ECS_EXCLUDE_UNTRACKED_IMAGE", "amazonlinux:2,amazonlinux:3")() + imagesNotDelete := parseImageCleanupExclusionList("ECS_EXCLUDE_UNTRACKED_IMAGE") +- expectedImages := []string{"amazonlinux:2", "amazonlinux:3", CachedImageNameAgentContainer, CachedImageNamePauseContainer} ++ expectedImages := []string{"amazonlinux:2", "amazonlinux:3"} + assert.Equal(t, expectedImages, imagesNotDelete, "unexpected imageCleanupExclusionList") + } + +diff --git a/agent/config/parse.go b/agent/config/parse.go +index 186f126d..ef1165af 100644 +--- a/agent/config/parse.go ++++ b/agent/config/parse.go +@@ -309,16 +309,11 @@ func parseImageCleanupExclusionList(envVar string) []string { + var imageCleanupExclusionList []string + if imageEnv == "" { + seelog.Debugf("Environment variable empty: %s", imageEnv) ++ return nil + } else { + imageCleanupExclusionList = strings.Split(imageEnv, ",") + } + +- // append known cached internal images to imageCleanupExclusionLis +- imageCleanupExclusionList = append(imageCleanupExclusionList, CachedImageNameAgentContainer, CachedImageNamePauseContainer) +- +- for _, image := range imageCleanupExclusionList { +- seelog.Infof("Image excluded from cleanup: %s", image) +- } + return imageCleanupExclusionList + } + +diff --git a/agent/engine/docker_image_manager.go b/agent/engine/docker_image_manager.go +index 2d575457..22815db5 100644 +--- a/agent/engine/docker_image_manager.go ++++ b/agent/engine/docker_image_manager.go +@@ -81,7 +81,7 @@ func NewImageManager(cfg *config.Config, client dockerapi.DockerClient, state do + numImagesToDelete: cfg.NumImagesToDeletePerCycle, + imageCleanupTimeInterval: cfg.ImageCleanupInterval, + imagePullBehavior: cfg.ImagePullBehavior, +- imageCleanupExclusionList: cfg.ImageCleanupExclusionList, ++ imageCleanupExclusionList: buildImageCleanupExclusionList(cfg), + deleteNonECSImagesEnabled: cfg.DeleteNonECSImagesEnabled, + nonECSContainerCleanupWaitDuration: cfg.TaskCleanupWaitDuration, + numNonECSContainersToDelete: cfg.NumNonECSContainersToDeletePerCycle, +@@ -89,6 +89,19 @@ func NewImageManager(cfg *config.Config, client dockerapi.DockerClient, state do + } + } + ++func buildImageCleanupExclusionList(cfg *config.Config) []string { ++ // append known cached internal images to imageCleanupExclusionList ++ excludedImages := append(cfg.ImageCleanupExclusionList, ++ cfg.PauseContainerImageName+":"+cfg.PauseContainerTag, ++ config.DefaultPauseContainerImageName+":"+config.DefaultPauseContainerTag, ++ config.CachedImageNameAgentContainer, ++ ) ++ for _, image := range excludedImages { ++ seelog.Infof("Image excluded from cleanup: %s", image) ++ } ++ return excludedImages ++} ++ + func (imageManager *dockerImageManager) SetSaver(stateManager statemanager.Saver) { + imageManager.saver = stateManager + } +diff --git a/agent/engine/docker_image_manager_test.go b/agent/engine/docker_image_manager_test.go +index 6fbacba4..8a014dc3 100644 +--- a/agent/engine/docker_image_manager_test.go ++++ b/agent/engine/docker_image_manager_test.go +@@ -44,6 +44,23 @@ func defaultTestConfig() *config.Config { + return cfg + } + ++func TestNewImageManagerExcludesCachedImages(t *testing.T) { ++ cfg := defaultTestConfig() ++ cfg.PauseContainerImageName = "pause-name" ++ cfg.PauseContainerTag = "pause-tag" ++ cfg.ImageCleanupExclusionList = []string{"excluded:1"} ++ expected := []string{ ++ "excluded:1", ++ "pause-name:pause-tag", ++ config.DefaultPauseContainerImageName + ":" + config.DefaultPauseContainerTag, ++ config.CachedImageNameAgentContainer, ++ } ++ imageManager := NewImageManager(cfg, nil, nil) ++ dockerImageManager, ok := imageManager.(*dockerImageManager) ++ require.True(t, ok, "imageManager must be *dockerImageManager") ++ assert.ElementsMatch(t, expected, dockerImageManager.imageCleanupExclusionList) ++} ++ + // TestImagePullRemoveDeadlock tests if there's a deadlock when trying to + // pull an image while image clean up is in progress + func TestImagePullRemoveDeadlock(t *testing.T) { +-- +2.27.0 + diff --git a/packages/ecs-agent/0002-bottlerocket-default-filesystem-locations.patch b/packages/ecs-agent/0002-bottlerocket-default-filesystem-locations.patch new file mode 100644 index 00000000000..010c0290a8b --- /dev/null +++ b/packages/ecs-agent/0002-bottlerocket-default-filesystem-locations.patch @@ -0,0 +1,82 @@ +From f3e8123fc244d256e95f0dac0a12ee8cf383d92a Mon Sep 17 00:00:00 2001 +From: Samuel Karp +Date: Wed, 8 Jul 2020 11:18:35 -0700 +Subject: [PATCH 1/2] bottlerocket: default filesystem locations + +--- + agent/config/config.go | 6 +++--- + agent/config/config_unix.go | 4 ++-- + agent/utils/license.go | 6 +++--- + 3 files changed, 8 insertions(+), 8 deletions(-) + +diff --git a/agent/config/config.go b/agent/config/config.go +index 1079684b..b11c0153 100644 +--- a/agent/config/config.go ++++ b/agent/config/config.go +@@ -51,7 +51,7 @@ const ( + AgentPrometheusExpositionPort = 51680 + + // defaultConfigFileName is the default (json-formatted) config file +- defaultConfigFileName = "/etc/ecs_container_agent/config.json" ++ defaultConfigFileName = "/etc/ecs/ecs.config.json" + + // DefaultClusterName is the name of the default cluster. + DefaultClusterName = "default" +@@ -115,13 +115,13 @@ const ( + minimumNumImagesToDeletePerCycle = 1 + + // defaultCNIPluginsPath is the default path where cni binaries are located +- defaultCNIPluginsPath = "/amazon-ecs-cni-plugins" ++ defaultCNIPluginsPath = "/usr/lib/amazon-ecs-agent/cni-plugins" + + // DefaultMinSupportedCNIVersion denotes the minimum version of cni spec required + DefaultMinSupportedCNIVersion = "0.3.0" + + // pauseContainerTarball is the path to the pause container tarball +- pauseContainerTarballPath = "/images/amazon-ecs-pause.tar" ++ pauseContainerTarballPath = "/usr/lib/amazon-ecs-agent/amazon-ecs-pause.tar" + + // DefaultTaskMetadataSteadyStateRate is set as 40. This is arrived from our benchmarking + // results where task endpoint can handle 4000 rps effectively. Here, 100 containers +diff --git a/agent/config/config_unix.go b/agent/config/config_unix.go +index ac5f5d0f..86984db4 100644 +--- a/agent/config/config_unix.go ++++ b/agent/config/config_unix.go +@@ -27,7 +27,7 @@ const ( + // AgentCredentialsAddress is used to serve the credentials for tasks. + AgentCredentialsAddress = "" // this is left blank right now for net=bridge + // defaultAuditLogFile specifies the default audit log filename +- defaultCredentialsAuditLogFile = "/log/audit.log" ++ defaultCredentialsAuditLogFile = "/var/log/ecs/audit.log" + // DefaultTaskCgroupPrefix is default cgroup prefix for ECS tasks + DefaultTaskCgroupPrefix = "/ecs" + +@@ -48,7 +48,7 @@ func DefaultConfig() Config { + DockerEndpoint: "unix:///var/run/docker.sock", + ReservedPorts: []uint16{SSHPort, DockerReservedPort, DockerReservedSSLPort, AgentIntrospectionPort, AgentCredentialsPort}, + ReservedPortsUDP: []uint16{}, +- DataDir: "/data/", ++ DataDir: "/var/lib/ecs/data", + DataDirOnHost: "/var/lib/ecs", + DisableMetrics: false, + ReservedMemory: 0, +diff --git a/agent/utils/license.go b/agent/utils/license.go +index 6eccff6a..324307cd 100644 +--- a/agent/utils/license.go ++++ b/agent/utils/license.go +@@ -24,9 +24,9 @@ type LicenseProvider interface { + type licenseProvider struct{} + + const ( +- licenseFile = "LICENSE" +- noticeFile = "NOTICE" +- thirdPartyFile = "THIRD-PARTY" ++ licenseFile = "/usr/share/licenses/ecs-agent/LICENSE" ++ noticeFile = "/usr/share/licenses/ecs-agent/NOTICE" ++ thirdPartyFile = "/usr/share/licenses/ecs-agent/THIRD-PARTY" + ) + + var readFile = ioutil.ReadFile +-- +2.27.0 + diff --git a/packages/ecs-agent/0003-bottlerocket-version-values-settable-with-linker.patch b/packages/ecs-agent/0003-bottlerocket-version-values-settable-with-linker.patch new file mode 100644 index 00000000000..27d4fba1f59 --- /dev/null +++ b/packages/ecs-agent/0003-bottlerocket-version-values-settable-with-linker.patch @@ -0,0 +1,30 @@ +From 648041259821e73e79965f3de3864109c3a5a816 Mon Sep 17 00:00:00 2001 +From: Samuel Karp +Date: Thu, 16 Jul 2020 10:52:01 -0700 +Subject: [PATCH] bottlerocket: version values settable with linker + +--- + agent/version/version.go | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/agent/version/version.go b/agent/version/version.go +index 4417a7cf..6fcb4832 100644 +--- a/agent/version/version.go ++++ b/agent/version/version.go +@@ -22,10 +22,10 @@ package version + // repository. Only the 'Version' const should change in checked-in source code + + // Version is the version of the Agent +-const Version = "1.41.0" ++var Version = "1.41.0" + + // GitDirty indicates the cleanliness of the git repo when this agent was built +-const GitDirty = true ++const GitDirty = false + + // GitShortHash is the short hash of this agent build +-const GitShortHash = "33c15e8b" ++var GitShortHash = "33c15e8b" +-- +2.27.0 + diff --git a/packages/ecs-agent/Cargo.toml b/packages/ecs-agent/Cargo.toml new file mode 100644 index 00000000000..96e91ae99e8 --- /dev/null +++ b/packages/ecs-agent/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "ecs-agent" +version = "0.1.0" +edition = "2018" +publish = false +build = "build.rs" + +[lib] +path = "pkg.rs" + +# ECS agent +[[package.metadata.build-package.external-files]] +url = "https://github.com/aws/amazon-ecs-agent/archive/v1.41.0.tar.gz" +sha512 = "d53d1a47eb41d39ffd541b6e416b42276f1d2207fdf5f031be173a8737655dab71ca0c88732716b6fee0761f7fede474cee08a21fd2db5d33b460d6aa547630e" + +# TODO: Package the CNI plugins +# The ECS agent repository includes two CNI plugins as git submodules. git +# archive does not include submodules, so the tarball above does not include +# the source of those plugins. + +[build-dependencies] +glibc = { path = "../glibc" } diff --git a/packages/ecs-agent/README.md b/packages/ecs-agent/README.md new file mode 100644 index 00000000000..f4a37c65afc --- /dev/null +++ b/packages/ecs-agent/README.md @@ -0,0 +1,27 @@ +# ecs-agent + +The ecs-agent package in Bottlerocket provides the ECS agent and a systemd unit +that sets up necessary configuration on the host. + +This README is temporary and is meant to track the known issues and remaining +work items for the ECS agent. + +## Known issues + +* The `docker` CLI is included in the variant. We should determine whether we + want to keep it or not; it's useful for debugging but it's not a + strictly-necessary component. +* CNI plugins are not yet packaged - This means that awsvpc mode and AppMesh + are both currently unsupported. + * The log path of CNI plugins - currently all of them are defaulting to the + container path of the log directory bind mount, e.g. + https://github.com/aws/amazon-ecs-cni-plugins/blob/0c6216c60401232805e50e31d4040ae84b0b23cf/plugins/eni/main.go#L32 + https://github.com/aws/amazon-ecs-agent/blob/master/agent/ecscni/plugin.go?rgh-link-date=2020-07-25T00%3A20%3A16Z#L39 + * The path to the process's network namespace handle is currently hardcoded + to the container path of the procfs bind mount + https://github.com/aws/amazon-ecs-agent/blob/782948476da6d995ad33c6a04130f8172820af27/agent/ecscni/types.go#L38 +* Logging is currently set to `debug` to assist with development. +* The systemd unit contains many `ExecStartPre`/`ExecStartPost` commands, with + little explanation or infrastructure. One `ExecStopPost` command depends on + `/bin/sh`, which is not available in Bottlerocket or compatible with our + goals for Bottlerocket. \ No newline at end of file diff --git a/packages/ecs-agent/build.rs b/packages/ecs-agent/build.rs new file mode 100644 index 00000000000..cad8999af53 --- /dev/null +++ b/packages/ecs-agent/build.rs @@ -0,0 +1,9 @@ +use std::process::{exit, Command}; + +fn main() -> Result<(), std::io::Error> { + let ret = Command::new("buildsys").arg("build-package").status()?; + if !ret.success() { + exit(1); + } + Ok(()) +} diff --git a/packages/ecs-agent/ecs-agent.spec b/packages/ecs-agent/ecs-agent.spec new file mode 100644 index 00000000000..94affea36ee --- /dev/null +++ b/packages/ecs-agent/ecs-agent.spec @@ -0,0 +1,99 @@ +%global goproject github.com/aws +%global gorepo amazon-ecs-agent +%global goimport %{goproject}/%{gorepo} + +%global gover 1.41.0 +# git rev-parse --short=8 +%global gitrev 3776bee9 + +Name: %{_cross_os}ecs-agent +Version: %{gover} +Release: 1%{?dist} +Summary: Amazon Elastic Container Service agent +License: Apache-2.0 +URL: https://%{goimport} +Source0: https://%{goimport}/archive/v%{gover}.tar.gz +Source1: ecs.service +Source2: ecs-tmpfiles.conf +Source3: pause-image-VERSION +Source4: pause-config.json +Source5: pause-manifest.json +Source6: pause-repositories + +# Upstream: https://github.com/aws/amazon-ecs-agent/pull/2513 +# Upstream status: Merged +Patch0001: 0001-engine-move-default-image-exclusions.patch + +# Bottlerocket-specific - filesystem location of the pause image +Patch0002: 0002-bottlerocket-default-filesystem-locations.patch + +# Bottlerocket-specific - version data can be set with linker options +Patch0003: 0003-bottlerocket-version-values-settable-with-linker.patch + +BuildRequires: %{_cross_os}glibc-devel + +Requires: %{_cross_os}docker-engine +# for sysctl +Requires: %{_cross_os}procps +Requires: %{_cross_os}iptables + +%description +%{summary}. + +%prep +%autosetup -Sgit -n %{gorepo}-%{gover} -p1 +%cross_go_setup %{gorepo}-%{gover} %{goproject} %{goimport} + +%build +# Build the agent +%cross_go_configure %{goimport} +PAUSE_CONTAINER_IMAGE_NAME="amazon/amazon-ecs-pause" +PAUSE_CONTAINER_IMAGE_TAG="bottlerocket" +LD_PAUSE_CONTAINER_NAME="-X github.com/aws/amazon-ecs-agent/agent/config.DefaultPauseContainerImageName=${PAUSE_CONTAINER_IMAGE_NAME}" +LD_PAUSE_CONTAINER_TAG="-X github.com/aws/amazon-ecs-agent/agent/config.DefaultPauseContainerTag=${PAUSE_CONTAINER_IMAGE_TAG}" +LD_VERSION="-X github.com/aws/amazon-ecs-agent/agent/version.Version=%{gover}" +LD_GIT_REV="-X github.com/aws/amazon-ecs-agent/agent/version.GitShortHash=%{gitrev}" +go build -a \ + -buildmode pie \ + -ldflags "${LD_PAUSE_CONTAINER_NAME} ${LD_PAUSE_CONTAINER_TAG} ${LD_VERSION} ${LD_GIT_REV}" \ + -o amazon-ecs-agent \ + ./agent + +# Build the pause container +( + set -x + cd misc/pause-container/buildPause + mkdir -p rootfs/usr/bin + make BIN=rootfs/usr/bin/pause GCC=%{_cross_triple}-musl-gcc CFLAGS="%{_cross_cflags} -static" + + # Construct image + mkdir -p image/rootfs + tar cvf image/rootfs/layer.tar -C rootfs . + DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//') + install -m 0644 %{S:3} image/rootfs/VERSION + install -m 0644 %{S:4} image/config.json + sed -i "s/~~digest~~/${DIGEST}/" image/config.json + install -m 0644 %{S:5} image/manifest.json + install -m 0644 %{S:6} image/repositories + tar cvf ../../../amazon-ecs-pause.tar -C image . +) + +%install +install -D -p -m 0755 amazon-ecs-agent %{buildroot}%{_cross_bindir}/amazon-ecs-agent +install -D -p -m 0644 amazon-ecs-pause.tar %{buildroot}%{_cross_libdir}/amazon-ecs-agent/amazon-ecs-pause.tar + +install -D -p -m 0644 %{S:1} %{buildroot}%{_cross_unitdir}/ecs.service +install -D -p -m 0644 %{S:2} %{buildroot}%{_cross_tmpfilesdir}/ecs.conf + +%cross_scan_attribution go-vendor agent/vendor + +%files +%{_cross_attribution_file} +%{_cross_attribution_vendor_dir} +%license LICENSE NOTICE THIRD-PARTY +%{_cross_bindir}/amazon-ecs-agent +%{_cross_unitdir}/ecs.service +%{_cross_tmpfilesdir}/ecs.conf +%{_cross_libdir}/amazon-ecs-agent/amazon-ecs-pause.tar + +%changelog diff --git a/packages/ecs-agent/ecs-tmpfiles.conf b/packages/ecs-agent/ecs-tmpfiles.conf new file mode 100644 index 00000000000..8bba088b1a1 --- /dev/null +++ b/packages/ecs-agent/ecs-tmpfiles.conf @@ -0,0 +1,2 @@ +d /var/lib/ecs/data 0700 root root +d /var/log/ecs 0755 root root diff --git a/packages/ecs-agent/ecs.service b/packages/ecs-agent/ecs.service new file mode 100644 index 00000000000..5261c728163 --- /dev/null +++ b/packages/ecs-agent/ecs.service @@ -0,0 +1,34 @@ +[Unit] +Description=Amazon Elastic Container Service - container agent +Documentation=https://aws.amazon.com/documentation/ecs/ +Requires=docker.service +After=docker.service configured.target +Wants=network-online.target configured.target + +[Service] +Type=simple +Restart=on-failure +RestartPreventExitStatus=5 +RestartSec=1s +EnvironmentFile=-/etc/ecs/ecs.config +Environment=ECS_LOGFILE=/var/log/ecs/ecs-agent.log ECS_LOGLEVEL=debug +Environment=ECS_CHECKPOINT=true +ExecStartPre=/sbin/iptables -t nat -A PREROUTING -d 169.254.170.2/32 \ + -p tcp -m tcp --dport 80 -j DNAT --to-destination 127.0.0.1:51679 +ExecStartPre=/sbin/iptables -t nat -A OUTPUT -d 169.254.170.2/32 \ + -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 51679 +ExecStartPre=/sbin/iptables -t filter -I INPUT --dst 127.0.0.0/8 ! \ + --src 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP +ExecStartPre=/sbin/sysctl -w net.ipv4.conf.all.route_localnet=1 +ExecStart=/usr/bin/amazon-ecs-agent +ExecStopPost=-/sbin/iptables -t nat -D PREROUTING -d 169.254.170.2/32 \ + -p tcp -m tcp --dport 80 -j DNAT --to-destination 127.0.0.1:51679 +ExecStopPost=-/sbin/iptables -t nat -D OUTPUT -d 169.254.170.2/32 \ + -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 51679 +ExecStopPost=-/sbin/iptables -t filter -D INPUT --dst 127.0.0.0/8 ! \ + --src 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP +ExecStopPost=/bin/sh -c \ + "/sbin/sysctl -w net.ipv4.conf.all.route_localnet=$(/sbin/sysctl -q -n net.ipv4.conf.default.route_localnet)" + +[Install] +WantedBy=multi-user.target diff --git a/packages/ecs-agent/pause-config.json b/packages/ecs-agent/pause-config.json new file mode 100644 index 00000000000..989255f38cb --- /dev/null +++ b/packages/ecs-agent/pause-config.json @@ -0,0 +1 @@ +{"author":"Amazon Web Services, Inc.","config":{"Cmd":["/usr/bin/pause"],"ArgsEscaped":true},"created":"2014-12-12T01:12:53.332832423Z","history":[{"created":"2014-12-12T01:12:53.332832423Z","author":"Amazon Web Services, Inc.","created_by":"[] + [] === \"\"","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:~~digest~~"]}} diff --git a/packages/ecs-agent/pause-image-VERSION b/packages/ecs-agent/pause-image-VERSION new file mode 100644 index 00000000000..9f8e9b69a33 --- /dev/null +++ b/packages/ecs-agent/pause-image-VERSION @@ -0,0 +1 @@ +1.0 \ No newline at end of file diff --git a/packages/ecs-agent/pause-manifest.json b/packages/ecs-agent/pause-manifest.json new file mode 100644 index 00000000000..514fbec19cd --- /dev/null +++ b/packages/ecs-agent/pause-manifest.json @@ -0,0 +1 @@ +[{"Config":"config.json","RepoTags":["amazon/amazon-ecs-pause:bottlerocket"],"Layers":["rootfs/layer.tar"]}] diff --git a/packages/ecs-agent/pause-repositories b/packages/ecs-agent/pause-repositories new file mode 100644 index 00000000000..47a8eeb37ac --- /dev/null +++ b/packages/ecs-agent/pause-repositories @@ -0,0 +1 @@ +{"amazon/amazon-ecs-pause":{"bottlerocket":"rootfs"}} diff --git a/packages/ecs-agent/pkg.rs b/packages/ecs-agent/pkg.rs new file mode 100644 index 00000000000..a711fc02eee --- /dev/null +++ b/packages/ecs-agent/pkg.rs @@ -0,0 +1 @@ +// not used \ No newline at end of file From a7a7971ec9af20e62acb011df0486221329061b2 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 16 Jun 2020 12:12:34 -0700 Subject: [PATCH 06/28] docker-engine: use cgroupfs cgroup driver The systemd cgroup driver imposes additional restrictions on the name of cgroup parents, causing workflows that depend on non-conforming names to fail. --- packages/docker-engine/daemon.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/docker-engine/daemon.json b/packages/docker-engine/daemon.json index ce8c1096617..d0f1ad2127c 100644 --- a/packages/docker-engine/daemon.json +++ b/packages/docker-engine/daemon.json @@ -4,7 +4,7 @@ "live-restore": true, "max-concurrent-downloads": 10, "storage-driver": "overlay2", - "exec-opts": ["native.cgroupdriver=systemd"], + "exec-opts": ["native.cgroupdriver=cgroupfs"], "data-root": "/var/lib/docker", "selinux-enabled": true } From 169b4d056d77b45920550146f6f26c2c9acafbfb Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 8 Jul 2020 22:02:44 -0700 Subject: [PATCH 07/28] aws-ecs-1: add ecs-agent package --- variants/aws-ecs-1/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/variants/aws-ecs-1/Cargo.toml b/variants/aws-ecs-1/Cargo.toml index b8d353d33df..8bceaaba440 100644 --- a/variants/aws-ecs-1/Cargo.toml +++ b/variants/aws-ecs-1/Cargo.toml @@ -14,6 +14,9 @@ included-packages = [ "docker-engine", "docker-init", "docker-proxy", + +# ecs + "ecs-agent" ] [lib] From 75d49e9e891b8a360ab0daa3e9151505c455777d Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Mon, 22 Jun 2020 14:22:07 -0700 Subject: [PATCH 08/28] packages: define %{_cross_variant} macro This macro can be used for conditions in variant-specific packages like `os`. --- Dockerfile | 1 + packages/README.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d32e924c574..626274c4882 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,6 +50,7 @@ ENV PACKAGE=${PACKAGE} ARCH=${ARCH} COPY ./macros/${ARCH} ./macros/shared ./macros/rust ./macros/cargo ./packages/${PACKAGE}/ . RUN rpmdev-setuptree \ && cat ${ARCH} shared rust cargo > .rpmmacros \ + && echo "%_cross_variant ${VARIANT}" >> .rpmmacros \ && rm ${ARCH} shared rust cargo \ && mv *.spec rpmbuild/SPECS \ && find . -maxdepth 1 -not -path '*/\.*' -type f -exec mv {} rpmbuild/SOURCES/ \; \ diff --git a/packages/README.md b/packages/README.md index 341b0ecc88b..d46868ebb21 100644 --- a/packages/README.md +++ b/packages/README.md @@ -185,7 +185,8 @@ Requires: %{name} Macros start with `%`. If the macro is specific to Bottlerocket, it will include the `cross` token. -The definitions for these can be found in [macros](../macros). +The definitions for most of these can be found in [macros](../macros). +The definition for `%{_cross_variant}` is the Bottlerocket variant being built. When developing a package on an RPM-based system, you can expand the macros with a command like this. ``` From 0e06a206c871c33f3ffa22d423b2c02c65a462ca Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 23 Jun 2020 11:27:28 -0700 Subject: [PATCH 09/28] ecs-settings-applier: initial implementation --- sources/Cargo.lock | 11 ++++ sources/Cargo.toml | 1 + sources/api/ecs-settings-applier/Cargo.toml | 15 +++++ sources/api/ecs-settings-applier/src/main.rs | 63 ++++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 sources/api/ecs-settings-applier/Cargo.toml create mode 100644 sources/api/ecs-settings-applier/src/main.rs diff --git a/sources/Cargo.lock b/sources/Cargo.lock index be54795e881..14e219c145e 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -747,6 +747,17 @@ dependencies = [ "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ecs-settings-applier" +version = "0.1.0" +dependencies = [ + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "models 0.1.0", + "schnauzer 0.1.0", + "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "either" version = "1.5.3" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 71f53aebd06..f7d10464f7c 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -4,6 +4,7 @@ members = [ "api/apiclient", "api/bork", "api/early-boot-config", + "api/ecs-settings-applier", "api/netdog", "api/sundog", "api/schnauzer", diff --git a/sources/api/ecs-settings-applier/Cargo.toml b/sources/api/ecs-settings-applier/Cargo.toml new file mode 100644 index 00000000000..0ccd8ab2e7f --- /dev/null +++ b/sources/api/ecs-settings-applier/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ecs-settings-applier" +version = "0.1.0" +authors = ["Samuel Karp "] +license = "Apache-2.0 OR MIT" +publish = false +edition = "2018" + +[dependencies] +serde = {version = "1.0", features = ["derive"]} +serde_json = "1" +models = { path = "../../models" } +schnauzer = { path = "../schnauzer" } +log = "0.4" + diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs new file mode 100644 index 00000000000..47a6b4d00f9 --- /dev/null +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -0,0 +1,63 @@ +use log::{debug, error}; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use serde::{Serialize}; + +// FIXME Get from configuration in the future +const DEFAULT_API_SOCKET: &str = "/run/api.sock"; + +#[derive(Serialize, Debug)] +#[serde(rename_all="PascalCase")] +struct ECSConfig { + #[serde(skip_serializing_if = "Option::is_none")] + cluster: Option, +} + +fn main() { + // Get all settings values for config file templates + debug!("Requesting settings values"); + let settings = match schnauzer::get_settings(&DEFAULT_API_SOCKET) { + Ok(s) => s, + Err(e) => { + error!("Failed to read settings: {}", e); + return + } + }; + + debug!("settings = {:#?}", settings.settings); + let config = ECSConfig{ cluster: settings.settings.and_then(|s| s.ecs).and_then(|s| s.cluster)}; + let serialized = serde_json::to_string(&config).unwrap(); + debug!("serialized = {}", serialized); + + let config_path = PathBuf::from("/etc/ecs/ecs.config.json"); + match write_to_disk(config_path, serialized) { + Some(e) => { + error!("Error! {}", e) + // TODO exit + } + _ => {} + } +} + +/// Writes the rendered data at the proper location +fn write_to_disk, C: AsRef<[u8]>>(path: P, contents: C) -> Option { + if let Some(dirname) = path.as_ref().parent() { + let result = fs::create_dir_all(dirname); + match result { + Err(e) => { + return Some(e) + } + _ => {} + }; + }; + + return match fs::write(path, contents) { + Err(e) => { + Some(e) + } + _ => { + None + } + } +} \ No newline at end of file From 75d4f04124917c6cdf3019dc682d6b6cb35acf13 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 23 Jun 2020 11:28:13 -0700 Subject: [PATCH 10/28] os: build ecs-settings-applier for aws-ecs-1 ecs-settings-applier is built conditionally for aws-ecs-1. Attempting to compile ecs-settings-applier for another variant will fail, as the applier depends on the definition of model::Model in the ECS variant. --- packages/os/os.spec | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/os/os.spec b/packages/os/os.spec index a9f77c4232c..9100deec79a 100644 --- a/packages/os/os.spec +++ b/packages/os/os.spec @@ -147,6 +147,13 @@ Summary: Thar data store migrations %description -n %{_cross_os}migrations %{summary}. +%if "%{_cross_variant}" == "aws-ecs-1" +%package -n %{_cross_os}ecs-settings-applier +Summary: Settings generator for ECS +%description -n %{_cross_os}ecs-settings-applier +%{summary}. +%endif + %prep %setup -T -c %cargo_prep @@ -178,6 +185,13 @@ mkdir bin -p apiclient \ %{nil} +# Build conditional ECS component +%if "%{_cross_variant}" == "aws-ecs-1" +%cargo_build --manifest-path %{_builddir}/sources/Cargo.toml \ + -p ecs-settings-applier \ + %{nil} +%endif + # Build the migrations for crate in $(find %{_builddir}/sources/api/migration/migrations -name 'Cargo.toml'); do %cargo_build_static --manifest-path "${crate}" @@ -196,6 +210,16 @@ do install -p -m 0755 ${HOME}/.cache/%{__cargo_target}/release/${p} %{buildroot}%{_cross_bindir} done + +%if "%{_cross_variant}" == "aws-ecs-1" +for p in \ + ecs-settings-applier; +do + install -p -m 0755 ${HOME}/.cache/%{__cargo_target}/release/${p} %{buildroot}%{_cross_bindir} +done +%endif + + for p in apiclient ; do install -p -m 0755 ${HOME}/.cache/%{__cargo_target_static}/release/${p} %{buildroot}%{_cross_bindir} done @@ -330,4 +354,9 @@ install -p -m 0644 %{S:202} %{buildroot}%{_cross_tmpfilesdir}/thar-be-updates.co %files -n %{_cross_os}logdog %{_cross_bindir}/logdog +%if "%{_cross_variant}" == "aws-ecs-1" +%files -n %{_cross_os}ecs-settings-applier +%{_cross_bindir}/ecs-settings-applier +%endif + %changelog From c9b34a17e64154d778fda812dad653244552f47b Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 23 Jun 2020 11:31:26 -0700 Subject: [PATCH 11/28] aws-ecs-1: use ecs-settings-applier --- sources/models/src/aws-ecs-1/override-defaults.toml | 10 +++++++++- variants/aws-ecs-1/Cargo.toml | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sources/models/src/aws-ecs-1/override-defaults.toml b/sources/models/src/aws-ecs-1/override-defaults.toml index e89c0ed270c..427f8f5772e 100644 --- a/sources/models/src/aws-ecs-1/override-defaults.toml +++ b/sources/models/src/aws-ecs-1/override-defaults.toml @@ -1,3 +1,11 @@ [configuration-files.containerd-config-toml] # No override to path -template-path = "/usr/share/templates/containerd-config-toml_aws-ecs-1" \ No newline at end of file +template-path = "/usr/share/templates/containerd-config-toml_aws-ecs-1" + +# ECS +[services.ecs] +restart-commands = ["/usr/bin/ecs-settings-applier", "/bin/systemctl try-reload-or-restart ecs.service"] +configuration-files = [] + +[metadata.settings.ecs] +affected-services = ["ecs"] diff --git a/variants/aws-ecs-1/Cargo.toml b/variants/aws-ecs-1/Cargo.toml index 8bceaaba440..130adfe9ff4 100644 --- a/variants/aws-ecs-1/Cargo.toml +++ b/variants/aws-ecs-1/Cargo.toml @@ -16,7 +16,8 @@ included-packages = [ "docker-proxy", # ecs - "ecs-agent" + "ecs-agent", + "ecs-settings-applier" ] [lib] From 8727e7846a9afeba1e074ec02916a865866a45fc Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 25 Jun 2020 12:23:50 -0700 Subject: [PATCH 12/28] docker-engine: allow Docker to use default bridge The default docker0 bridge allows container networking to function in a familiar way to developers. The bridge is used by `docker run` when exposing ports and used by Amazon ECS. --- packages/docker-engine/daemon.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/docker-engine/daemon.json b/packages/docker-engine/daemon.json index d0f1ad2127c..f0c106a9106 100644 --- a/packages/docker-engine/daemon.json +++ b/packages/docker-engine/daemon.json @@ -1,5 +1,4 @@ { - "bridge": "none", "log-driver": "journald", "live-restore": true, "max-concurrent-downloads": 10, From 44b649944bc296f0e6b5e2d6f454fc528a3f05f8 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 25 Jun 2020 18:03:28 -0700 Subject: [PATCH 13/28] ecs-settings-applier: use snafu for errors --- sources/Cargo.lock | 1 + sources/api/ecs-settings-applier/Cargo.toml | 1 + sources/api/ecs-settings-applier/src/main.rs | 83 ++++++++++++-------- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 14e219c145e..db3797e62f1 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -756,6 +756,7 @@ dependencies = [ "schnauzer 0.1.0", "serde 1.0.111 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.53 (registry+https://github.com/rust-lang/crates.io-index)", + "snafu 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/sources/api/ecs-settings-applier/Cargo.toml b/sources/api/ecs-settings-applier/Cargo.toml index 0ccd8ab2e7f..4c7888c93c0 100644 --- a/sources/api/ecs-settings-applier/Cargo.toml +++ b/sources/api/ecs-settings-applier/Cargo.toml @@ -12,4 +12,5 @@ serde_json = "1" models = { path = "../../models" } schnauzer = { path = "../schnauzer" } log = "0.4" +snafu = "0.6" diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 47a6b4d00f9..8fea5ac4ab6 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -1,12 +1,16 @@ -use log::{debug, error}; +use log::debug; +use std::{process}; use std::fs; -use std::io; -use std::path::{Path, PathBuf}; +use std::path::{Path}; use serde::{Serialize}; +use snafu::{ResultExt}; // FIXME Get from configuration in the future const DEFAULT_API_SOCKET: &str = "/run/api.sock"; +const DEFAULT_ECS_CONFIG_PATH: &str = "/etc/ecs/ecs.config.json"; + + #[derive(Serialize, Debug)] #[serde(rename_all="PascalCase")] struct ECSConfig { @@ -14,50 +18,61 @@ struct ECSConfig { cluster: Option, } +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} + +fn run() -> Result<()> { // Get all settings values for config file templates debug!("Requesting settings values"); - let settings = match schnauzer::get_settings(&DEFAULT_API_SOCKET) { - Ok(s) => s, - Err(e) => { - error!("Failed to read settings: {}", e); - return - } - }; + let settings = schnauzer::get_settings(&DEFAULT_API_SOCKET).context(error::Settings)?; debug!("settings = {:#?}", settings.settings); let config = ECSConfig{ cluster: settings.settings.and_then(|s| s.ecs).and_then(|s| s.cluster)}; - let serialized = serde_json::to_string(&config).unwrap(); + let serialized = serde_json::to_string(&config).context(error::Serialization)?; debug!("serialized = {}", serialized); - let config_path = PathBuf::from("/etc/ecs/ecs.config.json"); - match write_to_disk(config_path, serialized) { - Some(e) => { - error!("Error! {}", e) - // TODO exit - } - _ => {} - } + write_to_disk(DEFAULT_ECS_CONFIG_PATH, serialized).context(error::FS{path:DEFAULT_ECS_CONFIG_PATH})?; + Ok(()) } /// Writes the rendered data at the proper location -fn write_to_disk, C: AsRef<[u8]>>(path: P, contents: C) -> Option { +fn write_to_disk, C: AsRef<[u8]>>(path: P, contents: C) -> std::io::Result<()> { if let Some(dirname) = path.as_ref().parent() { - let result = fs::create_dir_all(dirname); - match result { - Err(e) => { - return Some(e) - } - _ => {} - }; + fs::create_dir_all(dirname)?; }; - return match fs::write(path, contents) { - Err(e) => { - Some(e) - } - _ => { - None + fs::write(path, contents) +} + +type Result = std::result::Result; + +mod error { + use snafu::Snafu; + + #[derive(Debug, Snafu)] + #[snafu(visibility = "pub(super)")] + pub(super) enum Error { + #[snafu(display("Failed to read settings: {}", source))] + Settings{ + source: schnauzer::Error + }, + + #[snafu(display("Failed to serialize ECS config: {}", source))] + Serialization{ + source: serde_json::error::Error + }, + + #[snafu(display("Filesystem operation for path {} failed: {}", path, source))] + FS{ + path: &'static str, + source: std::io::Error } } -} \ No newline at end of file +} From 82d497d42a743d7c17dc04546781ac5d5dfc2832 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 26 Jun 2020 10:15:10 -0700 Subject: [PATCH 14/28] ecs-settings-applier: add --socket-path arg --- sources/api/ecs-settings-applier/src/main.rs | 51 ++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 8fea5ac4ab6..03dd1296304 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -1,13 +1,11 @@ use log::debug; -use std::{process}; +use std::{env, process}; use std::fs; use std::path::{Path}; use serde::{Serialize}; use snafu::{ResultExt}; -// FIXME Get from configuration in the future const DEFAULT_API_SOCKET: &str = "/run/api.sock"; - const DEFAULT_ECS_CONFIG_PATH: &str = "/etc/ecs/ecs.config.json"; @@ -29,9 +27,11 @@ fn main() { } fn run() -> Result<()> { + let args = parse_args(env::args()); + // Get all settings values for config file templates debug!("Requesting settings values"); - let settings = schnauzer::get_settings(&DEFAULT_API_SOCKET).context(error::Settings)?; + let settings = schnauzer::get_settings(&args.socket_path).context(error::Settings)?; debug!("settings = {:#?}", settings.settings); let config = ECSConfig{ cluster: settings.settings.and_then(|s| s.ecs).and_then(|s| s.cluster)}; @@ -51,6 +51,49 @@ fn write_to_disk, C: AsRef<[u8]>>(path: P, contents: C) -> std::i fs::write(path, contents) } +// Stores user-supplied arguments. +struct Args { + socket_path: String +} + +fn parse_args(args: env::Args) -> Args { + let mut socket_path = None; + let mut iter = args.skip(1); + while let Some(arg) = iter.next() { + match arg.as_ref() { + "--socket-path" => { + socket_path = Some( + iter.next() + .unwrap_or_else(|| usage_msg("Did not give argument to --socket-path")), + ) + } + _ => usage(), + } + } + Args { + socket_path: socket_path.unwrap_or_else(|| DEFAULT_API_SOCKET.to_string()), + } +} + +// Prints a more specific message before exiting through usage(). +fn usage_msg>(msg: S) -> ! { + eprintln!("{}\n", msg.as_ref()); + usage(); +} + +// Informs the user about proper usage of the program and exits. +fn usage() -> ! { + let program_name = env::args().next().unwrap_or_else(|| "program".to_string()); + eprintln!( + r"Usage: {} + [ (-s | --socket-path) PATH ] + + Socket path defaults to {}", + program_name, DEFAULT_API_SOCKET + ); + process::exit(2); +} + type Result = std::result::Result; mod error { From 2a8ea5110dd1563c11f8fba7a02ecb6d91b65e75 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 30 Jun 2020 17:28:54 -0700 Subject: [PATCH 15/28] ecs-settings-applier: add default attributes --- sources/api/ecs-settings-applier/src/main.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 03dd1296304..5a07a18fb12 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -7,6 +7,8 @@ use snafu::{ResultExt}; const DEFAULT_API_SOCKET: &str = "/run/api.sock"; const DEFAULT_ECS_CONFIG_PATH: &str = "/etc/ecs/ecs.config.json"; +const VARIANT_ATTRIBUTE_NAME: &str = "bottlerocket.variant"; +const VERSION_ATTRIBUTE_NAME: &str = "bottlerocket.version"; #[derive(Serialize, Debug)] @@ -14,6 +16,9 @@ const DEFAULT_ECS_CONFIG_PATH: &str = "/etc/ecs/ecs.config.json"; struct ECSConfig { #[serde(skip_serializing_if = "Option::is_none")] cluster: Option, + + #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] + instance_attributes: std::collections::HashMap, } // Returning a Result from main makes it print a Debug representation of the error, but with Snafu @@ -34,7 +39,17 @@ fn run() -> Result<()> { let settings = schnauzer::get_settings(&args.socket_path).context(error::Settings)?; debug!("settings = {:#?}", settings.settings); - let config = ECSConfig{ cluster: settings.settings.and_then(|s| s.ecs).and_then(|s| s.cluster)}; + let mut config = ECSConfig{ + cluster: settings.settings.and_then(|s| s.ecs).and_then(|s| s.cluster), + instance_attributes: std::collections::HashMap::new() + }; + match settings.os { + None => {} + Some(os) => { + config.instance_attributes.insert(VARIANT_ATTRIBUTE_NAME.to_string(), os.variant_id); + config.instance_attributes.insert(VERSION_ATTRIBUTE_NAME.to_string(), os.version_id.to_string()); + } + } let serialized = serde_json::to_string(&config).context(error::Serialization)?; debug!("serialized = {}", serialized); From cfcd781057575b105c1224e172fd24845cd11800 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 30 Jun 2020 18:46:46 -0700 Subject: [PATCH 16/28] models: add instance attributes for aws-ecs-1 --- packages/ecs-agent/README.md | 7 +- sources/models/src/lib.rs | 6 +- sources/models/src/modeled_types/ecs.rs | 177 ++++++++++++++++++++++++ sources/models/src/modeled_types/mod.rs | 2 + 4 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 sources/models/src/modeled_types/ecs.rs diff --git a/packages/ecs-agent/README.md b/packages/ecs-agent/README.md index f4a37c65afc..a35423e34b7 100644 --- a/packages/ecs-agent/README.md +++ b/packages/ecs-agent/README.md @@ -21,7 +21,6 @@ work items for the ECS agent. to the container path of the procfs bind mount https://github.com/aws/amazon-ecs-agent/blob/782948476da6d995ad33c6a04130f8172820af27/agent/ecscni/types.go#L38 * Logging is currently set to `debug` to assist with development. -* The systemd unit contains many `ExecStartPre`/`ExecStartPost` commands, with - little explanation or infrastructure. One `ExecStopPost` command depends on - `/bin/sh`, which is not available in Bottlerocket or compatible with our - goals for Bottlerocket. \ No newline at end of file +* The Bottlerocket datastore does not accept keys with `\` characters. This + means that even though `\` is valid in ECS attribute names, it is not + supported on Bottlerocket. diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index 6d9b450378d..5166f075182 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -81,8 +81,9 @@ use std::collections::HashMap; use std::net::Ipv4Addr; use crate::modeled_types::{ - DNSDomain, FriendlyVersion, KubernetesClusterName, KubernetesLabelKey, KubernetesLabelValue, - KubernetesTaintValue, SingleLineString, Url, ValidBase64, + DNSDomain, ECSAttributeKey, ECSAttributeValue, FriendlyVersion, KubernetesClusterName, + KubernetesLabelKey, KubernetesLabelValue, KubernetesTaintValue, SingleLineString, Url, + ValidBase64, }; // Kubernetes related settings. The dynamic settings are retrieved from @@ -108,6 +109,7 @@ struct KubernetesSettings { #[model] struct ECSSettings { cluster: String, + instance_attributes: HashMap, } // Update settings. Taken from userdata. The 'seed' setting is generated diff --git a/sources/models/src/modeled_types/ecs.rs b/sources/models/src/modeled_types/ecs.rs new file mode 100644 index 00000000000..862ed1cc13d --- /dev/null +++ b/sources/models/src/modeled_types/ecs.rs @@ -0,0 +1,177 @@ +use lazy_static::lazy_static; +use regex::Regex; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +// Just need serde's Error in scope to get its trait methods +use super::error; +use serde::de::Error as _; +use snafu::ensure; +use std::borrow::Borrow; +use std::convert::TryFrom; +use std::fmt; +use std::ops::Deref; + +/// ECSAttributeKey represents a string that contains a valid ECS attribute key. It stores +/// the original string and makes it accessible through standard traits. +// https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Attribute.html +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct ECSAttributeKey { + inner: String, +} + +// The name of the attribute. The name must contain between 1 and 128 +// characters and name may contain letters (uppercase and lowercase), numbers, +// hyphens, underscores, forward slashes, back slashes, or periods. +lazy_static! { + pub(crate) static ref ECS_ATTRIBUTE_KEY: Regex = Regex::new( + r"(?x)^ + [a-zA-Z0-9._/-]{1,128} + $" + ) + .unwrap(); +} + +impl TryFrom<&str> for ECSAttributeKey { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + ECS_ATTRIBUTE_KEY.is_match(input), + error::BigPattern { + thing: "ECS attribute key", + input + } + ); + Ok(ECSAttributeKey { + inner: input.to_string(), + }) + } +} + +string_impls_for!(ECSAttributeKey, "ECSAttributeKey"); + +#[cfg(test)] +mod test_ecs_attribute_key { + use super::ECSAttributeKey; + use std::convert::TryFrom; + + #[test] + fn good_keys() { + for key in &[ + "a", + "alphabetical", + "1234567890", + "with-dash", + "have.period/slash", + "have_underscore_too", + &"a".repeat(128), + ".leadingperiod", + "trailingperiod.", + ] { + ECSAttributeKey::try_from(*key).unwrap(); + } + } + + #[test] + fn bad_keys() { + for key in &[ + "", + &"a".repeat(129), + "@", + "$", + "%", + ":", + "no spaces allowed", + ] { + ECSAttributeKey::try_from(*key).unwrap_err(); + } + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// ECSAttributeValue represents a string that contains a valid ECS attribute value. It stores +/// the original string and makes it accessible through standard traits. +// https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Attribute.html +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct ECSAttributeValue { + inner: String, +} + +// The value of the attribute. The value must contain between 1 and 128 +// characters and may contain letters (uppercase and lowercase), numbers, +// hyphens, underscores, periods, at signs (@), forward slashes, back slashes, +// colons, or spaces. The value cannot contain any leading or trailing +// whitespace. +lazy_static! { + pub(crate) static ref ECS_ATTRIBUTE_VALUE: Regex = Regex::new( + r"(?x)^ + [a-zA-Z0-9.@:_/\\-] # at least one non-space + ( + ([a-zA-Z0-9.@:\ _/\\-]{0,126})? # spaces allowed + [a-zA-Z0-9.@:_/\\-] # end with non-space + )? + $" + ) + .unwrap(); +} + +impl TryFrom<&str> for ECSAttributeValue { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + ECS_ATTRIBUTE_VALUE.is_match(input), + error::BigPattern { + thing: "ECS attribute value", + input + } + ); + Ok(ECSAttributeValue { + inner: input.to_string(), + }) + } +} + +string_impls_for!(ECSAttributeValue, "ECSAttributeValue"); + +#[cfg(test)] +mod test_ecs_attribute_value { + use super::ECSAttributeValue; + use std::convert::TryFrom; + + #[test] + fn good_vals() { + for val in &[ + "a", + "alphabetical", + "1234567890", + "with-dash", + "have.period/slash", + "have/slash\\backslash", + "have_underscore_too", + "with spaces in between", + &"a".repeat(128), + ".leadingperiod", + "trailingperiod.", + "@ and : allowed too", + "\\", + "\\ \\", + ] { + ECSAttributeValue::try_from(*val).unwrap(); + } + } + + #[test] + fn bad_vals() { + for val in &[ + "", + &"a".repeat(129), + "$", + "%", + " leading space", + "trailing space ", + ] { + ECSAttributeValue::try_from(*val).unwrap_err(); + } + } +} diff --git a/sources/models/src/modeled_types/mod.rs b/sources/models/src/modeled_types/mod.rs index 86f43d56c5b..c7644960484 100644 --- a/sources/models/src/modeled_types/mod.rs +++ b/sources/models/src/modeled_types/mod.rs @@ -126,8 +126,10 @@ macro_rules! string_impls_for { } // Must be after macro definition +mod ecs; mod kubernetes; mod shared; +pub use ecs::*; pub use kubernetes::*; pub use shared::*; From 1d6ef4915b1394aa342e4b73867c06f2a28ee507 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 30 Jun 2020 23:35:49 -0700 Subject: [PATCH 17/28] ecs-settings-applier: render attributes --- sources/api/ecs-settings-applier/src/main.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 5a07a18fb12..2202e3a7404 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -39,8 +39,10 @@ fn run() -> Result<()> { let settings = schnauzer::get_settings(&args.socket_path).context(error::Settings)?; debug!("settings = {:#?}", settings.settings); + let ecs = settings.settings.and_then(|s| s.ecs); + let cluster = ecs.as_ref().and_then(|s| s.cluster.as_ref()); let mut config = ECSConfig{ - cluster: settings.settings.and_then(|s| s.ecs).and_then(|s| s.cluster), + cluster: cluster.map(|s| s.clone()), instance_attributes: std::collections::HashMap::new() }; match settings.os { @@ -50,6 +52,14 @@ fn run() -> Result<()> { config.instance_attributes.insert(VERSION_ATTRIBUTE_NAME.to_string(), os.version_id.to_string()); } } + match ecs.as_ref().and_then(|s| s.instance_attributes.as_ref()) { + None => {} + Some(attributes) => { + for (key, value) in attributes { + config.instance_attributes.insert(key.to_string(), value.to_string()); + } + } + } let serialized = serde_json::to_string(&config).context(error::Serialization)?; debug!("serialized = {}", serialized); From a0ad8bd1cabefadf318d08688dfd4b4c41e1a984 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 1 Jul 2020 13:02:25 -0700 Subject: [PATCH 18/28] aws-ecs-1: add setting for privileged containers --- sources/models/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index 5166f075182..eab3b181c2d 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -110,6 +110,7 @@ struct KubernetesSettings { struct ECSSettings { cluster: String, instance_attributes: HashMap, + allow_privileged_containers: bool, } // Update settings. Taken from userdata. The 'seed' setting is generated From b69acb53c8d00b9cdab954f2ba6266c6b51e6d49 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 1 Jul 2020 13:02:53 -0700 Subject: [PATCH 19/28] ecs-settings-applier: render privileged containers --- sources/api/ecs-settings-applier/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 2202e3a7404..35015ad5392 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -19,6 +19,9 @@ struct ECSConfig { #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] instance_attributes: std::collections::HashMap, + + #[serde(skip_serializing_if = "Option::is_none")] + privileged_disabled: Option, } // Returning a Result from main makes it print a Debug representation of the error, but with Snafu @@ -41,9 +44,11 @@ fn run() -> Result<()> { debug!("settings = {:#?}", settings.settings); let ecs = settings.settings.and_then(|s| s.ecs); let cluster = ecs.as_ref().and_then(|s| s.cluster.as_ref()); + let privileged_disabled = ecs.as_ref().and_then(|s| s.allow_privileged_containers).map(|s| !s); let mut config = ECSConfig{ cluster: cluster.map(|s| s.clone()), - instance_attributes: std::collections::HashMap::new() + instance_attributes: std::collections::HashMap::new(), + privileged_disabled, }; match settings.os { None => {} From 9d7b0974f077e26b0c4090558b02fcfae118fe16 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 1 Jul 2020 13:03:11 -0700 Subject: [PATCH 20/28] aws-ecs-1: default disable privileged containers --- sources/models/src/aws-ecs-1/override-defaults.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sources/models/src/aws-ecs-1/override-defaults.toml b/sources/models/src/aws-ecs-1/override-defaults.toml index 427f8f5772e..c78287256b4 100644 --- a/sources/models/src/aws-ecs-1/override-defaults.toml +++ b/sources/models/src/aws-ecs-1/override-defaults.toml @@ -9,3 +9,6 @@ configuration-files = [] [metadata.settings.ecs] affected-services = ["ecs"] + +[settings.ecs] +allow-privileged-containers = false From 061f2fdb4095777cc4234b737d32f47b8b51a8d3 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 16 Jul 2020 13:43:35 -0700 Subject: [PATCH 21/28] ecs-agent: build pause image archive reproducibly See https://reproducible-builds.org/docs/archives/ --- packages/ecs-agent/ecs-agent.spec | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ecs-agent/ecs-agent.spec b/packages/ecs-agent/ecs-agent.spec index 94affea36ee..2d6b03c9ece 100644 --- a/packages/ecs-agent/ecs-agent.spec +++ b/packages/ecs-agent/ecs-agent.spec @@ -6,6 +6,11 @@ # git rev-parse --short=8 %global gitrev 3776bee9 +# Construct reproducible tar archives +# See https://reproducible-builds.org/docs/archives/ +%global source_date_epoch 1234567890 +%global tar_cf tar --sort=name --mtime="@%{source_date_epoch}" --owner=0 --group=0 --numeric-owner -cf + Name: %{_cross_os}ecs-agent Version: %{gover} Release: 1%{?dist} @@ -68,14 +73,14 @@ go build -a \ # Construct image mkdir -p image/rootfs - tar cvf image/rootfs/layer.tar -C rootfs . + %tar_cf image/rootfs/layer.tar -C rootfs . DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//') install -m 0644 %{S:3} image/rootfs/VERSION install -m 0644 %{S:4} image/config.json sed -i "s/~~digest~~/${DIGEST}/" image/config.json install -m 0644 %{S:5} image/manifest.json install -m 0644 %{S:6} image/repositories - tar cvf ../../../amazon-ecs-pause.tar -C image . + %tar_cf ../../../amazon-ecs-pause.tar -C image . ) %install From 4587c6950c1a49e70c3d731eeef1b034b4c1d471 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 17 Jul 2020 14:36:42 -0700 Subject: [PATCH 22/28] .github: add aws-ecs-1 variant to Build workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8bc56809f8..aeb97fcae0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ jobs: continue-on-error: ${{ matrix.supported }} strategy: matrix: - variant: [aws-k8s-1.15, aws-k8s-1.16, aws-k8s-1.17] + variant: [aws-k8s-1.15, aws-k8s-1.16, aws-k8s-1.17, aws-ecs-1] arch: [x86_64, aarch64] supported: [true] include: From 0debe9f77567c6b4206cf38cd507d24a52d8a740 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 21 Jul 2020 15:28:09 -0700 Subject: [PATCH 23/28] ecs-settings-applier: destructure ecs settings --- sources/api/ecs-settings-applier/src/main.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 35015ad5392..2b0d469c69c 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -43,8 +43,13 @@ fn run() -> Result<()> { debug!("settings = {:#?}", settings.settings); let ecs = settings.settings.and_then(|s| s.ecs); - let cluster = ecs.as_ref().and_then(|s| s.cluster.as_ref()); - let privileged_disabled = ecs.as_ref().and_then(|s| s.allow_privileged_containers).map(|s| !s); + let ecs = if let Some(x) = ecs { + x + } else { + return Err(error::Error::Model); + }; + let cluster = ecs.cluster.as_ref(); + let privileged_disabled = ecs.allow_privileged_containers.map(|s| !s); let mut config = ECSConfig{ cluster: cluster.map(|s| s.clone()), instance_attributes: std::collections::HashMap::new(), @@ -57,7 +62,7 @@ fn run() -> Result<()> { config.instance_attributes.insert(VERSION_ATTRIBUTE_NAME.to_string(), os.version_id.to_string()); } } - match ecs.as_ref().and_then(|s| s.instance_attributes.as_ref()) { + match ecs.instance_attributes.as_ref() { None => {} Some(attributes) => { for (key, value) in attributes { @@ -137,6 +142,8 @@ mod error { source: schnauzer::Error }, + Model, + #[snafu(display("Failed to serialize ECS config: {}", source))] Serialization{ source: serde_json::error::Error From 3402fa7f3ac2fed50a7a26ea2cbec32457d42657 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Tue, 21 Jul 2020 16:12:51 -0700 Subject: [PATCH 24/28] ecs-agent: use systemd-sysctl and sysctl.d --- packages/ecs-agent/README.md | 4 ++++ packages/ecs-agent/ecs-agent.spec | 21 +++++++++++---------- packages/ecs-agent/ecs-sysctl.conf | 2 ++ packages/ecs-agent/ecs.service | 3 --- 4 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 packages/ecs-agent/ecs-sysctl.conf diff --git a/packages/ecs-agent/README.md b/packages/ecs-agent/README.md index a35423e34b7..39a0fd3ab4f 100644 --- a/packages/ecs-agent/README.md +++ b/packages/ecs-agent/README.md @@ -21,6 +21,10 @@ work items for the ECS agent. to the container path of the procfs bind mount https://github.com/aws/amazon-ecs-agent/blob/782948476da6d995ad33c6a04130f8172820af27/agent/ecscni/types.go#L38 * Logging is currently set to `debug` to assist with development. +* The systemd unit contains many `ExecStartPre`/`ExecStopPost` commands, with + little explanation or infrastructure. The `ExecStartPre` commands should + probably be run exactly once, and the `ExecStopPost` commands probably + shouldn't ever run. * The Bottlerocket datastore does not accept keys with `\` characters. This means that even though `\` is valid in ECS attribute names, it is not supported on Bottlerocket. diff --git a/packages/ecs-agent/ecs-agent.spec b/packages/ecs-agent/ecs-agent.spec index 2d6b03c9ece..29c20502edb 100644 --- a/packages/ecs-agent/ecs-agent.spec +++ b/packages/ecs-agent/ecs-agent.spec @@ -20,10 +20,11 @@ URL: https://%{goimport} Source0: https://%{goimport}/archive/v%{gover}.tar.gz Source1: ecs.service Source2: ecs-tmpfiles.conf -Source3: pause-image-VERSION -Source4: pause-config.json -Source5: pause-manifest.json -Source6: pause-repositories +Source3: ecs-sysctl.conf +Source4: pause-image-VERSION +Source5: pause-config.json +Source6: pause-manifest.json +Source7: pause-repositories # Upstream: https://github.com/aws/amazon-ecs-agent/pull/2513 # Upstream status: Merged @@ -38,8 +39,6 @@ Patch0003: 0003-bottlerocket-version-values-settable-with-linker.patch BuildRequires: %{_cross_os}glibc-devel Requires: %{_cross_os}docker-engine -# for sysctl -Requires: %{_cross_os}procps Requires: %{_cross_os}iptables %description @@ -75,11 +74,11 @@ go build -a \ mkdir -p image/rootfs %tar_cf image/rootfs/layer.tar -C rootfs . DIGEST=$(sha256sum image/rootfs/layer.tar | sed -e 's/ .*//') - install -m 0644 %{S:3} image/rootfs/VERSION - install -m 0644 %{S:4} image/config.json + install -m 0644 %{S:4} image/rootfs/VERSION + install -m 0644 %{S:5} image/config.json sed -i "s/~~digest~~/${DIGEST}/" image/config.json - install -m 0644 %{S:5} image/manifest.json - install -m 0644 %{S:6} image/repositories + install -m 0644 %{S:6} image/manifest.json + install -m 0644 %{S:7} image/repositories %tar_cf ../../../amazon-ecs-pause.tar -C image . ) @@ -89,6 +88,7 @@ install -D -p -m 0644 amazon-ecs-pause.tar %{buildroot}%{_cross_libdir}/amazon-e install -D -p -m 0644 %{S:1} %{buildroot}%{_cross_unitdir}/ecs.service install -D -p -m 0644 %{S:2} %{buildroot}%{_cross_tmpfilesdir}/ecs.conf +install -D -p -m 0644 %{S:3} %{buildroot}%{_cross_sysctldir}/90-ecs.conf %cross_scan_attribution go-vendor agent/vendor @@ -99,6 +99,7 @@ install -D -p -m 0644 %{S:2} %{buildroot}%{_cross_tmpfilesdir}/ecs.conf %{_cross_bindir}/amazon-ecs-agent %{_cross_unitdir}/ecs.service %{_cross_tmpfilesdir}/ecs.conf +%{_cross_sysctldir}/90-ecs.conf %{_cross_libdir}/amazon-ecs-agent/amazon-ecs-pause.tar %changelog diff --git a/packages/ecs-agent/ecs-sysctl.conf b/packages/ecs-agent/ecs-sysctl.conf new file mode 100644 index 00000000000..64c0759a606 --- /dev/null +++ b/packages/ecs-agent/ecs-sysctl.conf @@ -0,0 +1,2 @@ +# ECS task role endpoint +net.ipv4.conf.all.route_localnet = 1 diff --git a/packages/ecs-agent/ecs.service b/packages/ecs-agent/ecs.service index 5261c728163..b718e353854 100644 --- a/packages/ecs-agent/ecs.service +++ b/packages/ecs-agent/ecs.service @@ -19,7 +19,6 @@ ExecStartPre=/sbin/iptables -t nat -A OUTPUT -d 169.254.170.2/32 \ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 51679 ExecStartPre=/sbin/iptables -t filter -I INPUT --dst 127.0.0.0/8 ! \ --src 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP -ExecStartPre=/sbin/sysctl -w net.ipv4.conf.all.route_localnet=1 ExecStart=/usr/bin/amazon-ecs-agent ExecStopPost=-/sbin/iptables -t nat -D PREROUTING -d 169.254.170.2/32 \ -p tcp -m tcp --dport 80 -j DNAT --to-destination 127.0.0.1:51679 @@ -27,8 +26,6 @@ ExecStopPost=-/sbin/iptables -t nat -D OUTPUT -d 169.254.170.2/32 \ -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 51679 ExecStopPost=-/sbin/iptables -t filter -D INPUT --dst 127.0.0.0/8 ! \ --src 127.0.0.0/8 -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP -ExecStopPost=/bin/sh -c \ - "/sbin/sysctl -w net.ipv4.conf.all.route_localnet=$(/sbin/sysctl -q -n net.ipv4.conf.default.route_localnet)" [Install] WantedBy=multi-user.target From 950a0ff3d8661520cf15a21455751a4edc54a9fb Mon Sep 17 00:00:00 2001 From: Jacob Vallejo Date: Wed, 22 Jul 2020 19:10:04 +0000 Subject: [PATCH 25/28] Unpack expected Options --- sources/api/ecs-settings-applier/src/main.rs | 38 +++++++------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 2b0d469c69c..f716e4ce877 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -3,7 +3,7 @@ use std::{env, process}; use std::fs; use std::path::{Path}; use serde::{Serialize}; -use snafu::{ResultExt}; +use snafu::{OptionExt, ResultExt}; const DEFAULT_API_SOCKET: &str = "/run/api.sock"; const DEFAULT_ECS_CONFIG_PATH: &str = "/etc/ecs/ecs.config.json"; @@ -11,7 +11,7 @@ const VARIANT_ATTRIBUTE_NAME: &str = "bottlerocket.variant"; const VERSION_ATTRIBUTE_NAME: &str = "bottlerocket.version"; -#[derive(Serialize, Debug)] +#[derive(Serialize, Debug, Default)] #[serde(rename_all="PascalCase")] struct ECSConfig { #[serde(skip_serializing_if = "Option::is_none")] @@ -42,32 +42,20 @@ fn run() -> Result<()> { let settings = schnauzer::get_settings(&args.socket_path).context(error::Settings)?; debug!("settings = {:#?}", settings.settings); - let ecs = settings.settings.and_then(|s| s.ecs); - let ecs = if let Some(x) = ecs { - x - } else { - return Err(error::Error::Model); - }; - let cluster = ecs.cluster.as_ref(); - let privileged_disabled = ecs.allow_privileged_containers.map(|s| !s); + let ecs = settings.settings.and_then(|s| s.ecs).context(error::Model)?; + let mut config = ECSConfig{ - cluster: cluster.map(|s| s.clone()), - instance_attributes: std::collections::HashMap::new(), - privileged_disabled, + cluster: ecs.cluster.map(|s| s.to_owned()), + privileged_disabled: ecs.allow_privileged_containers.map(|s| !s), + ..Default::default() }; - match settings.os { - None => {} - Some(os) => { - config.instance_attributes.insert(VARIANT_ATTRIBUTE_NAME.to_string(), os.variant_id); - config.instance_attributes.insert(VERSION_ATTRIBUTE_NAME.to_string(), os.version_id.to_string()); - } + if let Some(os) = settings.os { + config.instance_attributes.insert(VARIANT_ATTRIBUTE_NAME.to_string(), os.variant_id); + config.instance_attributes.insert(VERSION_ATTRIBUTE_NAME.to_string(), os.version_id.to_string()); } - match ecs.instance_attributes.as_ref() { - None => {} - Some(attributes) => { - for (key, value) in attributes { - config.instance_attributes.insert(key.to_string(), value.to_string()); - } + if let Some(attributes) = ecs.instance_attributes { + for (key, value) in attributes { + config.instance_attributes.insert(key.to_string(), value.to_string()); } } let serialized = serde_json::to_string(&config).context(error::Serialization)?; From b95c80ff07b07991374f230830114ea8529ed77e Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 22 Jul 2020 15:31:49 -0700 Subject: [PATCH 26/28] ecs-settings-applier: add README with cargo-readme --- sources/Cargo.lock | 1 + sources/api/ecs-settings-applier/Cargo.toml | 5 +++ sources/api/ecs-settings-applier/README.md | 14 +++++++++ sources/api/ecs-settings-applier/README.tpl | 9 ++++++ sources/api/ecs-settings-applier/build.rs | 32 ++++++++++++++++++++ sources/api/ecs-settings-applier/src/main.rs | 12 ++++++-- 6 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 sources/api/ecs-settings-applier/README.md create mode 100644 sources/api/ecs-settings-applier/README.tpl create mode 100644 sources/api/ecs-settings-applier/build.rs diff --git a/sources/Cargo.lock b/sources/Cargo.lock index db3797e62f1..67e04a81735 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -751,6 +751,7 @@ dependencies = [ name = "ecs-settings-applier" version = "0.1.0" dependencies = [ + "cargo-readme 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "models 0.1.0", "schnauzer 0.1.0", diff --git a/sources/api/ecs-settings-applier/Cargo.toml b/sources/api/ecs-settings-applier/Cargo.toml index 4c7888c93c0..570b193e3ca 100644 --- a/sources/api/ecs-settings-applier/Cargo.toml +++ b/sources/api/ecs-settings-applier/Cargo.toml @@ -5,6 +5,9 @@ authors = ["Samuel Karp "] license = "Apache-2.0 OR MIT" publish = false edition = "2018" +build = "build.rs" +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] [dependencies] serde = {version = "1.0", features = ["derive"]} @@ -14,3 +17,5 @@ schnauzer = { path = "../schnauzer" } log = "0.4" snafu = "0.6" +[build-dependencies] +cargo-readme = "3.1" diff --git a/sources/api/ecs-settings-applier/README.md b/sources/api/ecs-settings-applier/README.md new file mode 100644 index 00000000000..2135a8533dd --- /dev/null +++ b/sources/api/ecs-settings-applier/README.md @@ -0,0 +1,14 @@ +# ecs-settings-applier + +Current version: 0.1.0 + +## Introduction + +ecs-settings-applier generates a configuration file for the ECS agent from Bottlerocket settings. + +The configuration file for ECS is a JSON-formatted document with conditionally-defined keys and +embedded lists. + +## Colophon + +This text was generated using from `README.tpl` [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/main.rs`. \ No newline at end of file diff --git a/sources/api/ecs-settings-applier/README.tpl b/sources/api/ecs-settings-applier/README.tpl new file mode 100644 index 00000000000..82592715222 --- /dev/null +++ b/sources/api/ecs-settings-applier/README.tpl @@ -0,0 +1,9 @@ +# {{crate}} + +Current version: {{version}} + +{{readme}} + +## Colophon + +This text was generated using from `README.tpl` [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/main.rs`. diff --git a/sources/api/ecs-settings-applier/build.rs b/sources/api/ecs-settings-applier/build.rs new file mode 100644 index 00000000000..49828a1c42f --- /dev/null +++ b/sources/api/ecs-settings-applier/build.rs @@ -0,0 +1,32 @@ +// Automatically generate README.md from rustdoc. + +use std::env; +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; + +fn main() { + // Check for environment variable "SKIP_README". If it is set, + // skip README generation + if env::var_os("SKIP_README").is_some() { + return; + } + + let mut source = File::open("src/main.rs").unwrap(); + let mut template = File::open("README.tpl").unwrap(); + + let content = cargo_readme::generate_readme( + &PathBuf::from("."), // root + &mut source, // source + Some(&mut template), // template + // The "add x" arguments don't apply when using a template. + true, // add title + false, // add badges + false, // add license + true, // indent headings + ) + .unwrap(); + + let mut readme = File::create("README.md").unwrap(); + readme.write_all(content.as_bytes()).unwrap(); +} diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index f716e4ce877..66c153194ae 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -1,5 +1,13 @@ -use log::debug; -use std::{env, process}; +/*! +# Introduction + +ecs-settings-applier generates a configuration file for the ECS agent from Bottlerocket settings. + +The configuration file for ECS is a JSON-formatted document with conditionally-defined keys and +embedded lists. +*/ + +use log::debug;use std::{env, process}; use std::fs; use std::path::{Path}; use serde::{Serialize}; From 50a427b10d0324b0bca0906fe2aa6db33630e159 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 22 Jul 2020 16:25:55 -0700 Subject: [PATCH 27/28] ecs-settings-applier: simplify ECSConfig cluster --- sources/api/ecs-settings-applier/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 66c153194ae..6a35a324004 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -53,8 +53,8 @@ fn run() -> Result<()> { let ecs = settings.settings.and_then(|s| s.ecs).context(error::Model)?; let mut config = ECSConfig{ - cluster: ecs.cluster.map(|s| s.to_owned()), - privileged_disabled: ecs.allow_privileged_containers.map(|s| !s), + cluster: ecs.cluster, + privileged_disabled: ecs.allow_privileged_containers.map(|s| !s), ..Default::default() }; if let Some(os) = settings.os { From 802580722e43215db3818c057a80d738cbbcea41 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 23 Jul 2020 18:09:15 -0700 Subject: [PATCH 28/28] ecs-settings-applier: cargo fmt --- sources/api/ecs-settings-applier/src/main.rs | 52 ++++++++++++-------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/sources/api/ecs-settings-applier/src/main.rs b/sources/api/ecs-settings-applier/src/main.rs index 6a35a324004..4d53540d639 100644 --- a/sources/api/ecs-settings-applier/src/main.rs +++ b/sources/api/ecs-settings-applier/src/main.rs @@ -7,20 +7,20 @@ The configuration file for ECS is a JSON-formatted document with conditionally-d embedded lists. */ -use log::debug;use std::{env, process}; -use std::fs; -use std::path::{Path}; -use serde::{Serialize}; +use log::debug; +use serde::Serialize; use snafu::{OptionExt, ResultExt}; +use std::fs; +use std::path::Path; +use std::{env, process}; const DEFAULT_API_SOCKET: &str = "/run/api.sock"; const DEFAULT_ECS_CONFIG_PATH: &str = "/etc/ecs/ecs.config.json"; const VARIANT_ATTRIBUTE_NAME: &str = "bottlerocket.variant"; const VERSION_ATTRIBUTE_NAME: &str = "bottlerocket.version"; - #[derive(Serialize, Debug, Default)] -#[serde(rename_all="PascalCase")] +#[serde(rename_all = "PascalCase")] struct ECSConfig { #[serde(skip_serializing_if = "Option::is_none")] cluster: Option, @@ -50,26 +50,38 @@ fn run() -> Result<()> { let settings = schnauzer::get_settings(&args.socket_path).context(error::Settings)?; debug!("settings = {:#?}", settings.settings); - let ecs = settings.settings.and_then(|s| s.ecs).context(error::Model)?; + let ecs = settings + .settings + .and_then(|s| s.ecs) + .context(error::Model)?; - let mut config = ECSConfig{ + let mut config = ECSConfig { cluster: ecs.cluster, privileged_disabled: ecs.allow_privileged_containers.map(|s| !s), ..Default::default() }; if let Some(os) = settings.os { - config.instance_attributes.insert(VARIANT_ATTRIBUTE_NAME.to_string(), os.variant_id); - config.instance_attributes.insert(VERSION_ATTRIBUTE_NAME.to_string(), os.version_id.to_string()); + config + .instance_attributes + .insert(VARIANT_ATTRIBUTE_NAME.to_string(), os.variant_id); + config.instance_attributes.insert( + VERSION_ATTRIBUTE_NAME.to_string(), + os.version_id.to_string(), + ); } if let Some(attributes) = ecs.instance_attributes { for (key, value) in attributes { - config.instance_attributes.insert(key.to_string(), value.to_string()); + config + .instance_attributes + .insert(key.to_string(), value.to_string()); } } let serialized = serde_json::to_string(&config).context(error::Serialization)?; debug!("serialized = {}", serialized); - write_to_disk(DEFAULT_ECS_CONFIG_PATH, serialized).context(error::FS{path:DEFAULT_ECS_CONFIG_PATH})?; + write_to_disk(DEFAULT_ECS_CONFIG_PATH, serialized).context(error::FS { + path: DEFAULT_ECS_CONFIG_PATH, + })?; Ok(()) } @@ -84,7 +96,7 @@ fn write_to_disk, C: AsRef<[u8]>>(path: P, contents: C) -> std::i // Stores user-supplied arguments. struct Args { - socket_path: String + socket_path: String, } fn parse_args(args: env::Args) -> Args { @@ -134,21 +146,21 @@ mod error { #[snafu(visibility = "pub(super)")] pub(super) enum Error { #[snafu(display("Failed to read settings: {}", source))] - Settings{ - source: schnauzer::Error + Settings { + source: schnauzer::Error, }, Model, #[snafu(display("Failed to serialize ECS config: {}", source))] - Serialization{ - source: serde_json::error::Error + Serialization { + source: serde_json::error::Error, }, #[snafu(display("Filesystem operation for path {} failed: {}", path, source))] - FS{ + FS { path: &'static str, - source: std::io::Error - } + source: std::io::Error, + }, } }