From 582f029daa3a7bedbc2a49e10505203048c398ed Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 25 Aug 2023 22:30:12 +0000 Subject: [PATCH 1/3] Add #337 #338 AMI and add new customDeviceMapping config with new benchmark agent Signed-off-by: Peter Zhu --- lib/ci-stack.ts | 1 + lib/compute/agent-node-config.ts | 7 +++--- lib/compute/agent-nodes.ts | 38 +++++++++++++++++++++++++++++--- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/lib/ci-stack.ts b/lib/ci-stack.ts index b9047d0..ff3729b 100644 --- a/lib/ci-stack.ts +++ b/lib/ci-stack.ts @@ -161,6 +161,7 @@ export class CIStack extends Stack { agentNode.AL2_ARM64_DOCKER_HOST, agentNode.AL2023_ARM64_DOCKER_HOST, agentNode.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST, + agentNode.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST_NEW_SPEC, agentNode.UBUNTU2004_X64_GRADLE_CHECK, agentNode.UBUNTU2004_X64_DOCKER_BUILDER, agentNode.MACOS12_X64_MULTI_HOST, diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts index 890355b..b838a4e 100644 --- a/lib/compute/agent-node-config.ts +++ b/lib/compute/agent-node-config.ts @@ -21,6 +21,7 @@ export interface AgentNodeProps { agentType: string; amiId: string; instanceType: string; + customDeviceMapping: string; workerLabelString: string; remoteUser: string; maxTotalUses: number; @@ -162,7 +163,7 @@ export class AgentNodeConfig { associatePublicIp: false, connectBySSHProcess: false, connectionStrategy: 'PRIVATE_IP', - customDeviceMapping: '/dev/xvda=:300:true:::encrypted', + customDeviceMapping: config.customDeviceMapping, deleteRootOnTermination: true, description: `jenkinsAgentNode-${config.workerLabelString}`, ebsEncryptRootVolume: 'ENCRYPTED', @@ -246,7 +247,7 @@ export class AgentNodeConfig { associatePublicIp: false, connectBySSHProcess: false, connectionStrategy: 'PRIVATE_IP', - customDeviceMapping: '/dev/sda1=:300:true:gp3::encrypted', + customDeviceMapping: config.customDeviceMapping, deleteRootOnTermination: true, description: `jenkinsAgentNode-${config.workerLabelString}`, ebsEncryptRootVolume: 'ENCRYPTED', @@ -311,7 +312,7 @@ export class AgentNodeConfig { associatePublicIp: false, connectBySSHProcess: false, connectionStrategy: 'PRIVATE_IP', - customDeviceMapping: '/dev/sda1=:600:true:::encrypted', + customDeviceMapping: config.customDeviceMapping, deleteRootOnTermination: true, description: `jenkinsAgentNode-${config.workerLabelString}`, ebsEncryptRootVolume: 'ENCRYPTED', diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts index 4ffae93..0192c94 100644 --- a/lib/compute/agent-nodes.ts +++ b/lib/compute/agent-nodes.ts @@ -26,6 +26,8 @@ export class AgentNodes { readonly AL2023_X64_DOCKER_HOST_BENCHMARK_TEST: AgentNodeProps; + readonly AL2023_X64_DOCKER_HOST_BENCHMARK_TEST_NEW_SPEC: AgentNodeProps; + readonly UBUNTU2004_X64_GRADLE_CHECK: AgentNodeProps; readonly UBUNTU2004_X64_DOCKER_BUILDER: AgentNodeProps; @@ -47,6 +49,7 @@ export class AgentNodes { constructor(stack: Stack) { this.AL2023_X64 = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-AL2023-X64-C54xlarge-Single-Host', instanceType: 'C54xlarge', remoteUser: 'ec2-user', @@ -60,6 +63,7 @@ export class AgentNodes { }; this.AL2_X64_DOCKER_HOST = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host', instanceType: 'C54xlarge', remoteUser: 'ec2-user', @@ -73,6 +77,7 @@ export class AgentNodes { }; this.AL2023_X64_DOCKER_HOST = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host', instanceType: 'C54xlarge', remoteUser: 'ec2-user', @@ -86,6 +91,7 @@ export class AgentNodes { }; this.AL2023_ARM64 = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Single-Host', instanceType: 'C6g4xlarge', remoteUser: 'ec2-user', @@ -99,6 +105,7 @@ export class AgentNodes { }; this.AL2_ARM64_DOCKER_HOST = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', instanceType: 'C6g4xlarge', remoteUser: 'ec2-user', @@ -112,6 +119,7 @@ export class AgentNodes { }; this.AL2023_ARM64_DOCKER_HOST = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-AL2023-Arm64-C6g4xlarge-Docker-Host', instanceType: 'C6g4xlarge', remoteUser: 'ec2-user', @@ -125,6 +133,7 @@ export class AgentNodes { }; this.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test', instanceType: 'M52xlarge', remoteUser: 'ec2-user', @@ -136,8 +145,23 @@ export class AgentNodes { + ' sudo dnf update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* --exclude=python* -y && docker ps', remoteFs: '/var/jenkins', }; + this.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST_NEW_SPEC = { + agentType: 'unix', + customDeviceMapping: '/dev/xvda=:600:true:::encrypted', + workerLabelString: 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test-New-Spec', + instanceType: 'C54xlarge', + remoteUser: 'ec2-user', + maxTotalUses: -1, + minimumNumberOfSpareInstances: 1, + numExecutors: 4, + amiId: 'ami-01dfbac890366ceda', + initScript: 'sudo dnf clean all && sudo rm -rf /var/cache/dnf && sudo dnf repolist &&' + + ' sudo dnf update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* --exclude=python* -y && docker ps', + remoteFs: '/var/jenkins', + }; this.UBUNTU2004_X64_GRADLE_CHECK = { agentType: 'unix', + customDeviceMapping: '/dev/sda1=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-M58xlarge-Single-Host', instanceType: 'M58xlarge', remoteUser: 'ubuntu', @@ -151,6 +175,7 @@ export class AgentNodes { }; this.UBUNTU2004_X64_DOCKER_BUILDER = { agentType: 'unix', + customDeviceMapping: '/dev/sda1=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder', instanceType: 'M52xlarge', remoteUser: 'ubuntu', @@ -164,6 +189,7 @@ export class AgentNodes { }; this.MACOS12_X64_MULTI_HOST = { agentType: 'mac', + customDeviceMapping: '/dev/sda1=:300:true:gp3::encrypted', workerLabelString: 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host', instanceType: 'Mac1Metal', remoteUser: 'ec2-user', @@ -176,6 +202,7 @@ export class AgentNodes { }; this.WINDOWS2019_X64 = { agentType: 'windows', + customDeviceMapping: '/dev/sda1=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host', instanceType: 'C54xlarge', remoteUser: 'Administrator', @@ -188,30 +215,33 @@ export class AgentNodes { }; this.WINDOWS2019_X64_DOCKER_HOST = { agentType: 'windows', + customDeviceMapping: '/dev/sda1=:600:true:::encrypted', workerLabelString: 'Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Host', instanceType: 'C54xlarge', remoteUser: 'Administrator', maxTotalUses: -1, minimumNumberOfSpareInstances: 2, numExecutors: 4, - amiId: 'ami-03f326f81e2b16ae3', + amiId: 'ami-043fc8c0db1626fd2', initScript: 'echo %USERNAME% && dockerd --register-service && net start docker && echo started docker deamon && docker ps', remoteFs: 'C:/Users/Administrator/jenkins', }; this.WINDOWS2019_X64_DOCKER_BUILDER = { agentType: 'windows', + customDeviceMapping: '/dev/sda1=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Builder', instanceType: 'C54xlarge', remoteUser: 'Administrator', maxTotalUses: -1, - minimumNumberOfSpareInstances: 0, + minimumNumberOfSpareInstances: 1, numExecutors: 1, - amiId: 'ami-03f326f81e2b16ae3', + amiId: 'ami-043fc8c0db1626fd2', initScript: 'echo %USERNAME% && dockerd --register-service && net start docker && echo started docker deamon && docker ps', remoteFs: 'C:/Users/Administrator/jenkins', }; this.WINDOWS2019_X64_GRADLE_CHECK = { agentType: 'windows', + customDeviceMapping: '/dev/sda1=:300:true:::encrypted', workerLabelString: 'Jenkins-Agent-Windows2019-X64-C524xlarge-Single-Host', instanceType: 'C524xlarge', remoteUser: 'Administrator', @@ -224,6 +254,7 @@ export class AgentNodes { }; this.AL2_X64_DEFAULT_AGENT = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Default-Agent-X64-C5xlarge-Single-Host', instanceType: 'C54xlarge', remoteUser: 'ec2-user', @@ -241,6 +272,7 @@ export class AgentNodes { }; this.AL2_ARM64_DEFAULT_AGENT = { agentType: 'unix', + customDeviceMapping: '/dev/xvda=:300:true:::encrypted', workerLabelString: 'Jenkins-Default-Agent-ARM64-C5xlarge-Single-Host', instanceType: 'C6g4xlarge', remoteUser: 'ec2-user', From 1398a894c15c5cef676d9ea132acf849d341a3f9 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 25 Aug 2023 22:36:48 +0000 Subject: [PATCH 2/3] More tweaks Signed-off-by: Peter Zhu --- lib/compute/agent-nodes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts index 0192c94..cef8880 100644 --- a/lib/compute/agent-nodes.ts +++ b/lib/compute/agent-nodes.ts @@ -148,7 +148,7 @@ export class AgentNodes { this.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST_NEW_SPEC = { agentType: 'unix', customDeviceMapping: '/dev/xvda=:600:true:::encrypted', - workerLabelString: 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test-New-Spec', + workerLabelString: 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host-Benchmark-Test-New-Spec', instanceType: 'C54xlarge', remoteUser: 'ec2-user', maxTotalUses: -1, From 2ab21b3528e31713babb31720cd0b0afe05f91bc Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 29 Aug 2023 17:45:49 +0000 Subject: [PATCH 3/3] Remove DOCKER_HOST from the agent lable and name Signed-off-by: Peter Zhu --- lib/ci-stack.ts | 4 ++-- lib/compute/agent-nodes.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/ci-stack.ts b/lib/ci-stack.ts index ff3729b..165ac3f 100644 --- a/lib/ci-stack.ts +++ b/lib/ci-stack.ts @@ -160,8 +160,8 @@ export class CIStack extends Stack { agentNode.AL2023_ARM64, agentNode.AL2_ARM64_DOCKER_HOST, agentNode.AL2023_ARM64_DOCKER_HOST, - agentNode.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST, - agentNode.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST_NEW_SPEC, + agentNode.AL2023_X64_BENCHMARK_TEST, + agentNode.AL2023_X64_BENCHMARK_TEST_NEW_SPEC, agentNode.UBUNTU2004_X64_GRADLE_CHECK, agentNode.UBUNTU2004_X64_DOCKER_BUILDER, agentNode.MACOS12_X64_MULTI_HOST, diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts index cef8880..76dbff3 100644 --- a/lib/compute/agent-nodes.ts +++ b/lib/compute/agent-nodes.ts @@ -24,9 +24,9 @@ export class AgentNodes { readonly AL2023_ARM64_DOCKER_HOST: AgentNodeProps; - readonly AL2023_X64_DOCKER_HOST_BENCHMARK_TEST: AgentNodeProps; + readonly AL2023_X64_BENCHMARK_TEST: AgentNodeProps; - readonly AL2023_X64_DOCKER_HOST_BENCHMARK_TEST_NEW_SPEC: AgentNodeProps; + readonly AL2023_X64_BENCHMARK_TEST_NEW_SPEC: AgentNodeProps; readonly UBUNTU2004_X64_GRADLE_CHECK: AgentNodeProps; @@ -131,10 +131,10 @@ export class AgentNodes { + ' sudo dnf update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* --exclude=python* -y && docker ps', remoteFs: '/var/jenkins', }; - this.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST = { + this.AL2023_X64_BENCHMARK_TEST = { agentType: 'unix', customDeviceMapping: '/dev/xvda=:300:true:::encrypted', - workerLabelString: 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Benchmark-Test', + workerLabelString: 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test', instanceType: 'M52xlarge', remoteUser: 'ec2-user', maxTotalUses: -1, @@ -145,10 +145,10 @@ export class AgentNodes { + ' sudo dnf update --skip-broken --exclude=openssh* --exclude=docker* --exclude=gh* --exclude=python* -y && docker ps', remoteFs: '/var/jenkins', }; - this.AL2023_X64_DOCKER_HOST_BENCHMARK_TEST_NEW_SPEC = { + this.AL2023_X64_BENCHMARK_TEST_NEW_SPEC = { agentType: 'unix', customDeviceMapping: '/dev/xvda=:600:true:::encrypted', - workerLabelString: 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host-Benchmark-Test-New-Spec', + workerLabelString: 'Jenkins-Agent-AL2023-X64-C54xlarge-Benchmark-Test-New-Spec', instanceType: 'C54xlarge', remoteUser: 'ec2-user', maxTotalUses: -1,