From de97dccfbeff3507cb04d6ee77a3da4a49e0f0b8 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Mon, 18 Mar 2024 12:19:08 +0100 Subject: [PATCH 01/13] Integrate ccip-scripts deployment with devspace env --- charts/chainlink-cluster/README.md | 9 ++ charts/chainlink-cluster/devspace.yaml | 2 + .../templates/ccip-scripts-job.yaml | 38 ++++++ .../templates/ccip-scripts-pod.yaml | 87 ++++++++++++++ charts/chainlink-cluster/values.yaml | 113 ++++++++++++++++++ 5 files changed, 249 insertions(+) create mode 100644 charts/chainlink-cluster/templates/ccip-scripts-job.yaml create mode 100644 charts/chainlink-cluster/templates/ccip-scripts-pod.yaml diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 0943ca5a8b..4ffe960084 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -106,11 +106,20 @@ kubectl create ns cl-cluster kubectl config set-context --current --namespace cl-cluster ``` +Template +``` +helm template app -f values.yaml -n crib-radek . --output-dir .rendered \ + --set=ingress.baseDomain="$DEVSPACE_INGRESS_BASE_DOMAIN" +``` + Install ``` + helm install -f values.yaml cl-cluster . ``` + + ## Create a new release Bump version in `Chart.yml` add your changes and add `helm_release` label to any PR to trigger a release diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index e7cf75d493..4a9fff95ca 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -71,6 +71,8 @@ pipelines: --set=helm.values.chainlink.nodes[3].image=$image \ --set=helm.values.chainlink.nodes[4].image=$image \ --set=helm.values.chainlink.nodes[5].image=$image + --set=helm.values.ingress.baseDomain=$DEVSPACE_INGRESS_BASE_DOMAIN + --set=helm.values.ccip.ccipScriptsImage=$DEVSPACE_CCIP_SCRIPTS_IMAGE echo echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}" echo "To extend the TTL for e.g. 72 hours, run: devspace run ttl ${DEVSPACE_NAMESPACE} 72h" diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml new file mode 100644 index 0000000000..0ebf05c95f --- /dev/null +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ccip-scripts-config +data: + config.json: | + { + "hello": "world" + } +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ccip-scripts-deploy +spec: + template: + spec: + containers: + - name: ccip-scripts + image: {{.Values.ccip.ccipScriptsImage}} + command: ["cat", "version"] + volumeMounts: + - name: config-volume + mountPath: /data/config.json + securityContext: + capabilities: + drop: + - ALL + runAsUser: 999 + runAsGroup: 999 + runAsNonRoot: true + restartPolicy: Never + volumes: + - name: config-volume + configMap: + name: ccip-scripts-config + backoffLimit: 4 + diff --git a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml new file mode 100644 index 0000000000..55b54f047c --- /dev/null +++ b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ccip-scripts-config +data: + config.json: | + { + "EnvName": "{{$.Release.Namespace}}", + "DONCreds": { + "Env": "{{$.Release.Namespace}}", + "Bootstrap": { + {{- with (index $.Values.chainlink.nodes 0) }} + {{- $nameWithoutHyphen := .name | replace "-" "" }} + "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", + "Email": "notreal@fakeemail.ch", + "Password": "fj293fbBnlQ!f9vNs", + "InternalIP": "{{$.Release.Name}}-{{.name}}", + {{- end}} + "HTTPTimeout": null + }, + "Nodes": [ + {{- range $index, $cfg := $.Values.chainlink.nodes }} + {{- if ne $index 0}} + {{- $nameWithoutHyphen := $cfg.name | replace "-" "" }} + { + "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", + "Email": "notreal@fakeemail.ch", + "Password": "fj293fbBnlQ!f9vNs", + "InternalIP": "{{$.Release.Name}}-{{$cfg.name}}", + "HTTPTimeout": null + }, + {{- end}} + {{- end}} + ] + }, + {{- $networkIDs := list }} + {{- range $index, $cfg :=$.Values.ccip.chains }} + {{- $networkIDs = append $networkIDs $cfg.NetworkId }} + {{- end }} + + {{- $delimiter := "," }} + {{- $foldedString := join $delimiter $networkIDs }} + "ChainPairs":[{{- printf $foldedString | quote }}], + "CCIPChains": { + {{- range $index, $cfg := $.Values.ccip.chains }} + "geth_{{$cfg.NetworkId}}":{ + "NetworkURL": "wss://{{$.Release.Namespace}}-geth-{{$cfg.NetworkId}}-ws.{{$.Values.ingress.baseDomain}}", + "WalletKey": "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + "DeployLink": false, + "DeployWETH": false, + "ChainConfig": {{$cfg.ChainConfig | toJson}} + }, + {{- end}} + }, + "LaneDeploySettings": {{$.Values.ccip.LaneDeploySettings | toJson }} + } + +--- +apiVersion: v1 +kind: Pod +metadata: + name: ccip-scripts-deploy + +spec: + containers: + - name: ccip-scripts + image: {{.Values.ccip.ccipScriptsImage}} + command: + - "tail" + - "-f" + - "/dev/null" + volumeMounts: + - name: config-volume + mountPath: /data/config.json + securityContext: + capabilities: + drop: + - ALL + runAsUser: 999 + runAsGroup: 999 + runAsNonRoot: true + restartPolicy: Never + volumes: + - name: config-volume + configMap: + name: ccip-scripts-config + diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index b0866574c9..aeea5dbdd6 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -142,6 +142,118 @@ geth: limits: cpu: 1 memory: 1024Mi + +ccip: + ccipScriptsImage: ccip-scripts-mage:image-tag + chains: + - NetworkId: 1337 + WalletKey: 59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d + DeployLink: false + DeployWETH: false + ChainConfig: + EvmChainId: 1337 + GasSettings: + EIP1559: true + GasTipCap: 1000000000 + SupportedTokens: + ChainLink Token: + ChainId: 1337 + Token: "0x5fbdb2315678afecb367f032d93f642f64180aa3" + Pool: "0xa513e6e4b8f2a923d98304ec87f64353c4d5c853" + TokenPriceType: "TokenPrices" + Price: 10000000000000000000 + Aggregator: "0x0000000000000000000000000000000000000000" + TokenPoolType: "lockRelease" + WETH: + ChainId: 1337 + Token: "0xcf7ed3acca5a467e9e704c703e8d87f634fb0fc9" + Pool: "0x0000000000000000000000000000000000000000" + TokenPriceType: "TokenPrices" + Price: 1.8e+21 + Aggregator: "0x0000000000000000000000000000000000000000" + TokenPoolType: "feeTokenOnly" + Router: "0x0165878a594ca255338adfa4d48449f69242eb8f" + ARM: "0xdc64a140aa3e981100a9beca4e685f962f0cf6c9" + ARMProxy: "0x5fc8d32690cc91d4c39d9d3abcbd16989f875707" + PriceRegistry: "0x8a791620dd6260079bf849dc5567adc3f2fdc318" + DeploySettings: + DeployedAtBlock: 478 + TunableChainValues: + FinalityDepth: 1 + OptimisticConfirmations: 1 + BatchGasLimit: 7000000 + RelativeBoostPerWaitHour: 2400 + FeeUpdateHeartBeat: "24h0m0s" + FeeUpdateDeviationPPB: 100000000 + MaxGasPrice: 1000000000000 + InflightCacheExpiry: "3m0s" + RootSnoozeTime: "5m0s" + - NetworkId: 2337 + WalletKey: "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d" + DeployLink: false + DeployWETH: false + ChainConfig: + EvmChainId: 2337 + GasSettings: + EIP1559: true + GasTipCap: 1000000000 + SupportedTokens: + ChainLink Token: + ChainId: 2337 + Token: "0x8464135c8f25da09e49bc8782676a84730c318bc" + Pool: "0x1275d096b9dbf2347bd2a131fb6bdab0b4882487" + TokenPriceType: "TokenPrices" + Price: 10000000000000000000 + Aggregator: "0x0000000000000000000000000000000000000000" + TokenPoolType: "lockRelease" + WETH: + ChainId: 2337 + Token: "0x712516e61c8b383df4a63cfe83d7701bce54b03e" + Pool: "0x0000000000000000000000000000000000000000" + TokenPriceType: "TokenPrices" + Price: 1.8e+21 + Aggregator: "0x0000000000000000000000000000000000000000" + TokenPoolType: "feeTokenOnly" + Router: "0xc6ba8c3233ecf65b761049ef63466945c362edd2" + ARM: "0xbcf26943c0197d2ee0e5d05c716be60cc2761508" + ARMProxy: "0x59f2f1fcfe2474fd5f0b9ba1e73ca90b143eb8d0" + PriceRegistry: "0x0b48af34f4c854f5ae1a3d587da471fea45bad52" + DeploySettings: + DeployedAtBlock: 479 + TunableChainValues: + FinalityDepth: 1 + OptimisticConfirmations: 1 + BatchGasLimit: 7000000 + RelativeBoostPerWaitHour: 2400 + FeeUpdateHeartBeat: "24h0m0s" + FeeUpdateDeviationPPB: 100000000 + MaxGasPrice: 1000000000000 + InflightCacheExpiry: "3m0s" + RootSnoozeTime: "5m0s" + LaneDeploySettings: + geth_1337,geth_2337: + DeployLane: false + DeployPingPongDapp: false + CCIPLaneConfigs: + geth_1337: + geth_2337: + OnRamp: "0x610178da211fef7d417bc0e6fed39f05609ad788" + OffRamp: "0x0b306bf915c4d645ff596e518faf3f9669b97016" + CommitStore: "0x0dcd1bf9a1b36ce34237eeafef220932846bcd82" + PingPongDapp: "0x68b1d87f95878fe05b998f19b66f4baba5de1aed" + DeploySettings: + DeployedAtBlock: 478 + geth_2337: + geth_1337: + OnRamp: "0x0f5d1ef48f12b6f691401bfe88c2037c690a6afe" + OffRamp: "0x381445710b5e73d34af196c53a3d5cda58edbf7a" + CommitStore: "0x2de080e97b0cae9825375d31f5d0ed5751fdf16d" + PingPongDapp: "0x0d4ff719551e23185aeb16ffbf2abebb90635942" + DeploySettings: + DeployedAtBlock: 479 + + + # mockserver is https://www.mock-server.com/where/kubernetes.html # used to stub External Adapters mockserver: @@ -270,6 +382,7 @@ ingress: enabled: false annotations: {} ingressClassName: alb + baseDomain: "example.com" hosts: - host: chainlink-node-1.local http: From 6182b8c04c98c1e583efc08d64321812864f283c Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Tue, 19 Mar 2024 11:22:19 +0100 Subject: [PATCH 02/13] adjust ccip chain configs to match the latest structure from ccip-scripts repo Fix json syntax errors --- charts/chainlink-cluster/.gitignore | 1 + charts/chainlink-cluster/README.md | 3 +- charts/chainlink-cluster/devspace.yaml | 7 +- .../templates/ccip-scripts-pod.yaml | 23 ++-- charts/chainlink-cluster/values.yaml | 114 ++++++------------ 5 files changed, 57 insertions(+), 91 deletions(-) diff --git a/charts/chainlink-cluster/.gitignore b/charts/chainlink-cluster/.gitignore index 3ee791f740..0b1ce86a15 100644 --- a/charts/chainlink-cluster/.gitignore +++ b/charts/chainlink-cluster/.gitignore @@ -1,3 +1,4 @@ # Helm charts/ requirements.lock +.rendered \ No newline at end of file diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 4ffe960084..cdfbb6cd0f 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -109,7 +109,8 @@ kubectl config set-context --current --namespace cl-cluster Template ``` helm template app -f values.yaml -n crib-radek . --output-dir .rendered \ - --set=ingress.baseDomain="$DEVSPACE_INGRESS_BASE_DOMAIN" + --set=ingress.baseDomain="$DEVSPACE_INGRESS_BASE_DOMAIN" \ + --set=ccip.ccipScriptsImage=$DEVSPACE_CCIP_SCRIPTS_IMAGE ``` Install diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 4a9fff95ca..5df107ffed 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -17,6 +17,9 @@ vars: source: env # Time to wait for pods to be in `Ready` condition DEVSPACE_K8S_POD_WAIT_TIMEOUT: 600s + # Image URI required for deploying CCIP Contracts and Jobs + DEVSPACE_CCIP_SCRIPTS_IMAGE: + source: env # This is a list of `pipelines` that DevSpace can execute (you can define your own) pipelines: @@ -70,8 +73,8 @@ pipelines: --set=helm.values.chainlink.nodes[2].image=$image \ --set=helm.values.chainlink.nodes[3].image=$image \ --set=helm.values.chainlink.nodes[4].image=$image \ - --set=helm.values.chainlink.nodes[5].image=$image - --set=helm.values.ingress.baseDomain=$DEVSPACE_INGRESS_BASE_DOMAIN + --set=helm.values.chainlink.nodes[5].image=$image \ + --set=helm.values.ingress.baseDomain=$DEVSPACE_INGRESS_BASE_DOMAIN \ --set=helm.values.ccip.ccipScriptsImage=$DEVSPACE_CCIP_SCRIPTS_IMAGE echo echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}" diff --git a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml index 55b54f047c..3634b391fa 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml @@ -22,20 +22,20 @@ data: {{- range $index, $cfg := $.Values.chainlink.nodes }} {{- if ne $index 0}} {{- $nameWithoutHyphen := $cfg.name | replace "-" "" }} - { + {{- if ne $index 1 }},{{- end }}{ "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", "Email": "notreal@fakeemail.ch", "Password": "fj293fbBnlQ!f9vNs", "InternalIP": "{{$.Release.Name}}-{{$cfg.name}}", "HTTPTimeout": null - }, + } {{- end}} {{- end}} ] }, {{- $networkIDs := list }} {{- range $index, $cfg :=$.Values.ccip.chains }} - {{- $networkIDs = append $networkIDs $cfg.NetworkId }} + {{- $networkIDs = append $networkIDs $cfg.ChainId }} {{- end }} {{- $delimiter := "," }} @@ -43,13 +43,13 @@ data: "ChainPairs":[{{- printf $foldedString | quote }}], "CCIPChains": { {{- range $index, $cfg := $.Values.ccip.chains }} - "geth_{{$cfg.NetworkId}}":{ + {{- if ne $index 0 }},{{- end }}"{{$cfg.ChainId}}":{ "NetworkURL": "wss://{{$.Release.Namespace}}-geth-{{$cfg.NetworkId}}-ws.{{$.Values.ingress.baseDomain}}", - "WalletKey": "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", - "DeployLink": false, - "DeployWETH": false, + "WalletKey": {{$cfg.WalletKey | quote}}, + "DeployLink": {{$cfg.DeployLink}}, + "DeployWETH": {{$cfg.DeployWETH}}, "ChainConfig": {{$cfg.ChainConfig | toJson}} - }, + } {{- end}} }, "LaneDeploySettings": {{$.Values.ccip.LaneDeploySettings | toJson }} @@ -65,13 +65,16 @@ spec: containers: - name: ccip-scripts image: {{.Values.ccip.ccipScriptsImage}} + env: + - name: CONFIG_JSON_PATH + value: /data/config.json command: - "tail" - "-f" - "/dev/null" volumeMounts: - name: config-volume - mountPath: /data/config.json + mountPath: /data securityContext: capabilities: drop: @@ -84,4 +87,6 @@ spec: - name: config-volume configMap: name: ccip-scripts-config + - name: build-volume + emptyDir: {} diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index aeea5dbdd6..9d86678ee5 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -147,110 +147,66 @@ ccip: ccipScriptsImage: ccip-scripts-mage:image-tag chains: - NetworkId: 1337 - WalletKey: 59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d - DeployLink: false - DeployWETH: false + ChainId: geth_1337 + WalletKey: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + DeployLink: true + DeployWETH: true ChainConfig: EvmChainId: 1337 GasSettings: EIP1559: true GasTipCap: 1000000000 - SupportedTokens: - ChainLink Token: - ChainId: 1337 - Token: "0x5fbdb2315678afecb367f032d93f642f64180aa3" - Pool: "0xa513e6e4b8f2a923d98304ec87f64353c4d5c853" - TokenPriceType: "TokenPrices" - Price: 10000000000000000000 - Aggregator: "0x0000000000000000000000000000000000000000" - TokenPoolType: "lockRelease" - WETH: - ChainId: 1337 - Token: "0xcf7ed3acca5a467e9e704c703e8d87f634fb0fc9" - Pool: "0x0000000000000000000000000000000000000000" - TokenPriceType: "TokenPrices" - Price: 1.8e+21 - Aggregator: "0x0000000000000000000000000000000000000000" - TokenPoolType: "feeTokenOnly" - Router: "0x0165878a594ca255338adfa4d48449f69242eb8f" - ARM: "0xdc64a140aa3e981100a9beca4e685f962f0cf6c9" - ARMProxy: "0x5fc8d32690cc91d4c39d9d3abcbd16989f875707" - PriceRegistry: "0x8a791620dd6260079bf849dc5567adc3f2fdc318" + FeeTokens: + - "ChainLink Token" + - "WETH" + WrappedNative: "WETH" + Router: "0x0000000000000000000000000000000000000000" + ARM: "0x0000000000000000000000000000000000000000" + ARMProxy: "0x0000000000000000000000000000000000000000" + PriceRegistry: "0x0000000000000000000000000000000000000000" DeploySettings: - DeployedAtBlock: 478 + DeployARM: true + DeployTokenPools: true + DeployRouter: true + DeployPriceRegistry: true TunableChainValues: FinalityDepth: 1 OptimisticConfirmations: 1 - BatchGasLimit: 7000000 - RelativeBoostPerWaitHour: 2400 - FeeUpdateHeartBeat: "24h0m0s" - FeeUpdateDeviationPPB: 100000000 - MaxGasPrice: 1000000000000 - InflightCacheExpiry: "3m0s" - RootSnoozeTime: "5m0s" + MaxGasPrice: 200000000000 - NetworkId: 2337 + ChainId: geth_2337 WalletKey: "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d" - DeployLink: false - DeployWETH: false + DeployLink: true + DeployWETH: true ChainConfig: EvmChainId: 2337 GasSettings: EIP1559: true GasTipCap: 1000000000 - SupportedTokens: - ChainLink Token: - ChainId: 2337 - Token: "0x8464135c8f25da09e49bc8782676a84730c318bc" - Pool: "0x1275d096b9dbf2347bd2a131fb6bdab0b4882487" - TokenPriceType: "TokenPrices" - Price: 10000000000000000000 - Aggregator: "0x0000000000000000000000000000000000000000" - TokenPoolType: "lockRelease" - WETH: - ChainId: 2337 - Token: "0x712516e61c8b383df4a63cfe83d7701bce54b03e" - Pool: "0x0000000000000000000000000000000000000000" - TokenPriceType: "TokenPrices" - Price: 1.8e+21 - Aggregator: "0x0000000000000000000000000000000000000000" - TokenPoolType: "feeTokenOnly" - Router: "0xc6ba8c3233ecf65b761049ef63466945c362edd2" - ARM: "0xbcf26943c0197d2ee0e5d05c716be60cc2761508" - ARMProxy: "0x59f2f1fcfe2474fd5f0b9ba1e73ca90b143eb8d0" - PriceRegistry: "0x0b48af34f4c854f5ae1a3d587da471fea45bad52" + FeeTokens: + - "ChainLink Token" + - "WETH" + WrappedNative: "WETH" + Router: "0x0000000000000000000000000000000000000000" + ARM: "0x0000000000000000000000000000000000000000" + ARMProxy: "0x0000000000000000000000000000000000000000" + PriceRegistry: "0x0000000000000000000000000000000000000000" DeploySettings: - DeployedAtBlock: 479 + DeployARM: true + DeployTokenPools: true + DeployRouter: true + DeployPriceRegistry: true TunableChainValues: FinalityDepth: 1 OptimisticConfirmations: 1 - BatchGasLimit: 7000000 - RelativeBoostPerWaitHour: 2400 - FeeUpdateHeartBeat: "24h0m0s" - FeeUpdateDeviationPPB: 100000000 - MaxGasPrice: 1000000000000 - InflightCacheExpiry: "3m0s" - RootSnoozeTime: "5m0s" + MaxGasPrice: 200000000000 LaneDeploySettings: geth_1337,geth_2337: - DeployLane: false - DeployPingPongDapp: false + DeployLane: true + DeployPingPongDapp: true CCIPLaneConfigs: geth_1337: - geth_2337: - OnRamp: "0x610178da211fef7d417bc0e6fed39f05609ad788" - OffRamp: "0x0b306bf915c4d645ff596e518faf3f9669b97016" - CommitStore: "0x0dcd1bf9a1b36ce34237eeafef220932846bcd82" - PingPongDapp: "0x68b1d87f95878fe05b998f19b66f4baba5de1aed" - DeploySettings: - DeployedAtBlock: 478 geth_2337: - geth_1337: - OnRamp: "0x0f5d1ef48f12b6f691401bfe88c2037c690a6afe" - OffRamp: "0x381445710b5e73d34af196c53a3d5cda58edbf7a" - CommitStore: "0x2de080e97b0cae9825375d31f5d0ed5751fdf16d" - PingPongDapp: "0x0d4ff719551e23185aeb16ffbf2abebb90635942" - DeploySettings: - DeployedAtBlock: 479 From 2c65545b04978f7ef4ce705b43f436e567ce9639 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Tue, 19 Mar 2024 13:39:12 +0100 Subject: [PATCH 03/13] extract config map to separate file --- charts/chainlink-cluster/README.md | 11 +--- .../templates/ccip-scripts-cm.yaml | 56 +++++++++++++++++ .../templates/ccip-scripts-job.yaml | 22 +++---- .../templates/ccip-scripts-pod.yaml | 63 +------------------ 4 files changed, 68 insertions(+), 84 deletions(-) create mode 100644 charts/chainlink-cluster/templates/ccip-scripts-cm.yaml diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index cdfbb6cd0f..8d7044ceb4 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -106,20 +106,13 @@ kubectl create ns cl-cluster kubectl config set-context --current --namespace cl-cluster ``` -Template +Install ``` -helm template app -f values.yaml -n crib-radek . --output-dir .rendered \ +helm install -f values.yaml cl-cluster . \ --set=ingress.baseDomain="$DEVSPACE_INGRESS_BASE_DOMAIN" \ --set=ccip.ccipScriptsImage=$DEVSPACE_CCIP_SCRIPTS_IMAGE ``` -Install -``` - -helm install -f values.yaml cl-cluster . -``` - - ## Create a new release Bump version in `Chart.yml` add your changes and add `helm_release` label to any PR to trigger a release diff --git a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml new file mode 100644 index 0000000000..36475e68e2 --- /dev/null +++ b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ccip-scripts-config +data: + config.json: | + { + "EnvName": "{{$.Release.Namespace}}", + "DONCreds": { + "Env": "{{$.Release.Namespace}}", + "Bootstrap": { + {{- with (index $.Values.chainlink.nodes 0) }} + {{- $nameWithoutHyphen := .name | replace "-" "" }} + "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", + "Email": "notreal@fakeemail.ch", + "Password": "fj293fbBnlQ!f9vNs", + "InternalIP": "{{$.Release.Name}}-{{.name}}", + {{- end}} + "HTTPTimeout": null + }, + "Nodes": [ + {{- range $index, $cfg := $.Values.chainlink.nodes }} + {{- if ne $index 0}} + {{- $nameWithoutHyphen := $cfg.name | replace "-" "" }} + {{- if ne $index 1 }},{{- end }}{ + "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", + "Email": "notreal@fakeemail.ch", + "Password": "fj293fbBnlQ!f9vNs", + "InternalIP": "{{$.Release.Name}}-{{$cfg.name}}", + "HTTPTimeout": null + } + {{- end}} + {{- end}} + ] + }, + {{- $networkIDs := list }} + {{- range $index, $cfg :=$.Values.ccip.chains }} + {{- $networkIDs = append $networkIDs $cfg.ChainId }} + {{- end }} + + {{- $delimiter := "," }} + {{- $foldedString := join $delimiter $networkIDs }} + "ChainPairs":[{{- printf $foldedString | quote }}], + "CCIPChains": { + {{- range $index, $cfg := $.Values.ccip.chains }} + {{- if ne $index 0 }},{{- end }}"{{$cfg.ChainId}}":{ + "NetworkURL": "wss://{{$.Release.Namespace}}-geth-{{$cfg.NetworkId}}-ws.{{$.Values.ingress.baseDomain}}", + "WalletKey": {{$cfg.WalletKey | quote}}, + "DeployLink": {{$cfg.DeployLink}}, + "DeployWETH": {{$cfg.DeployWETH}}, + "ChainConfig": {{$cfg.ChainConfig | toJson}} + } + {{- end}} + }, + "LaneDeploySettings": {{$.Values.ccip.LaneDeploySettings | toJson }} + } diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index 0ebf05c95f..2e82fbfeea 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -1,13 +1,4 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ccip-scripts-config -data: - config.json: | - { - "hello": "world" - } ---- +{{- if ne .Values.ccip.debug true }} apiVersion: batch/v1 kind: Job metadata: @@ -18,10 +9,13 @@ spec: containers: - name: ccip-scripts image: {{.Values.ccip.ccipScriptsImage}} - command: ["cat", "version"] + env: + - name: CONFIG_JSON_PATH + value: /data/config.json + command: [ "go", "test", "pilot/crib_test.go" ] volumeMounts: - name: config-volume - mountPath: /data/config.json + mountPath: /data securityContext: capabilities: drop: @@ -34,5 +28,5 @@ spec: - name: config-volume configMap: name: ccip-scripts-config - backoffLimit: 4 - + backoffLimit: 1 +{{- end }} diff --git a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml index 3634b391fa..4d99c5aed5 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml @@ -1,61 +1,4 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ccip-scripts-config -data: - config.json: | - { - "EnvName": "{{$.Release.Namespace}}", - "DONCreds": { - "Env": "{{$.Release.Namespace}}", - "Bootstrap": { - {{- with (index $.Values.chainlink.nodes 0) }} - {{- $nameWithoutHyphen := .name | replace "-" "" }} - "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", - "Email": "notreal@fakeemail.ch", - "Password": "fj293fbBnlQ!f9vNs", - "InternalIP": "{{$.Release.Name}}-{{.name}}", - {{- end}} - "HTTPTimeout": null - }, - "Nodes": [ - {{- range $index, $cfg := $.Values.chainlink.nodes }} - {{- if ne $index 0}} - {{- $nameWithoutHyphen := $cfg.name | replace "-" "" }} - {{- if ne $index 1 }},{{- end }}{ - "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", - "Email": "notreal@fakeemail.ch", - "Password": "fj293fbBnlQ!f9vNs", - "InternalIP": "{{$.Release.Name}}-{{$cfg.name}}", - "HTTPTimeout": null - } - {{- end}} - {{- end}} - ] - }, - {{- $networkIDs := list }} - {{- range $index, $cfg :=$.Values.ccip.chains }} - {{- $networkIDs = append $networkIDs $cfg.ChainId }} - {{- end }} - - {{- $delimiter := "," }} - {{- $foldedString := join $delimiter $networkIDs }} - "ChainPairs":[{{- printf $foldedString | quote }}], - "CCIPChains": { - {{- range $index, $cfg := $.Values.ccip.chains }} - {{- if ne $index 0 }},{{- end }}"{{$cfg.ChainId}}":{ - "NetworkURL": "wss://{{$.Release.Namespace}}-geth-{{$cfg.NetworkId}}-ws.{{$.Values.ingress.baseDomain}}", - "WalletKey": {{$cfg.WalletKey | quote}}, - "DeployLink": {{$cfg.DeployLink}}, - "DeployWETH": {{$cfg.DeployWETH}}, - "ChainConfig": {{$cfg.ChainConfig | toJson}} - } - {{- end}} - }, - "LaneDeploySettings": {{$.Values.ccip.LaneDeploySettings | toJson }} - } - ---- +{{if .Values.ccip.debug }} apiVersion: v1 kind: Pod metadata: @@ -87,6 +30,4 @@ spec: - name: config-volume configMap: name: ccip-scripts-config - - name: build-volume - emptyDir: {} - +{{end}} \ No newline at end of file From 6618ce41d87f0921eafb127f6e6d7a027d355794 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Tue, 19 Mar 2024 14:02:04 +0100 Subject: [PATCH 04/13] setup network policies for ccip-scripts-deployer --- charts/chainlink-cluster/templates/ccip-scripts-job.yaml | 6 +++++- charts/chainlink-cluster/templates/ccip-scripts-pod.yaml | 5 ++++- .../templates/chainlink-db-networkpolicy.yaml | 4 ---- .../templates/chainlink-node-networkpolicy.yaml | 3 ++- charts/chainlink-cluster/templates/geth-networkpolicy.yaml | 4 ++-- .../templates/mockserver-networkpolicy.yaml | 4 ++-- charts/chainlink-cluster/values.yaml | 2 +- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index 2e82fbfeea..c5d68a2acb 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -3,6 +3,10 @@ apiVersion: batch/v1 kind: Job metadata: name: ccip-scripts-deploy + labels: + app: ccip-scripts-deployer + annotations: + prometheus.io/scrape: 'true' spec: template: spec: @@ -12,7 +16,7 @@ spec: env: - name: CONFIG_JSON_PATH value: /data/config.json - command: [ "go", "test", "pilot/crib_test.go" ] + command: [ "go", "test", "pilot/crib_test.go", "-v" ] volumeMounts: - name: config-volume mountPath: /data diff --git a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml index 4d99c5aed5..9c9e146d80 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml @@ -3,7 +3,10 @@ apiVersion: v1 kind: Pod metadata: name: ccip-scripts-deploy - + labels: + app: ccip-scripts-deployer + annotations: + prometheus.io/scrape: 'true' spec: containers: - name: ccip-scripts diff --git a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml index 5f7e7706ce..3e4c9f49b4 100644 --- a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml @@ -15,10 +15,6 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} - # Allow all runner pods to access the database pods. - - podSelector: - matchLabels: - app: runner ports: - protocol: TCP port: 5432 diff --git a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml index e63759a994..fe6df3cf0f 100644 --- a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml @@ -15,7 +15,8 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} + # Allow traffic from ccip-scripts-deployer - podSelector: matchLabels: - app: runner + app: ccip-scripts-deployer {{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml index 025d618450..38d0f07445 100644 --- a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml @@ -15,10 +15,10 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} - # Allow http and websocket connections from the runner pods. + # Allow traffic from ccip-scripts-deployer - podSelector: matchLabels: - app: runner + app: ccip-scripts-deployer ports: - protocol: TCP port: 8544 diff --git a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml index 6ac4f658e3..2efa7d2ed8 100644 --- a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml @@ -15,10 +15,10 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} - # Allow http traffic from the runner pods. + # Allow traffic from ccip-scripts-deployer - podSelector: matchLabels: - app: runner + app: ccip-scripts-deployer ports: - protocol: TCP port: 1080 diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index 9d86678ee5..9d7980db0c 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -144,7 +144,7 @@ geth: memory: 1024Mi ccip: - ccipScriptsImage: ccip-scripts-mage:image-tag + ccipScriptsImage: ccip-scripts-image:image-tag chains: - NetworkId: 1337 ChainId: geth_1337 From a7311b1fe326270b138a75dbf59cb458b662e8eb Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Tue, 19 Mar 2024 14:04:32 +0100 Subject: [PATCH 05/13] cleanup --- .../templates/ccip-scripts-job.yaml | 2 -- .../templates/ccip-scripts-pod.yaml | 36 ------------------- 2 files changed, 38 deletions(-) delete mode 100644 charts/chainlink-cluster/templates/ccip-scripts-pod.yaml diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index c5d68a2acb..156eb0c3fe 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -1,4 +1,3 @@ -{{- if ne .Values.ccip.debug true }} apiVersion: batch/v1 kind: Job metadata: @@ -33,4 +32,3 @@ spec: configMap: name: ccip-scripts-config backoffLimit: 1 -{{- end }} diff --git a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml b/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml deleted file mode 100644 index 9c9e146d80..0000000000 --- a/charts/chainlink-cluster/templates/ccip-scripts-pod.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{if .Values.ccip.debug }} -apiVersion: v1 -kind: Pod -metadata: - name: ccip-scripts-deploy - labels: - app: ccip-scripts-deployer - annotations: - prometheus.io/scrape: 'true' -spec: - containers: - - name: ccip-scripts - image: {{.Values.ccip.ccipScriptsImage}} - env: - - name: CONFIG_JSON_PATH - value: /data/config.json - command: - - "tail" - - "-f" - - "/dev/null" - volumeMounts: - - name: config-volume - mountPath: /data - securityContext: - capabilities: - drop: - - ALL - runAsUser: 999 - runAsGroup: 999 - runAsNonRoot: true - restartPolicy: Never - volumes: - - name: config-volume - configMap: - name: ccip-scripts-config -{{end}} \ No newline at end of file From 0f6a254b230af8c66d74e5580fee0136b73c5cc2 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Tue, 19 Mar 2024 17:14:19 +0100 Subject: [PATCH 06/13] use cluster local dns addressess instead of ingress --- charts/chainlink-cluster/templates/ccip-scripts-cm.yaml | 7 +++---- charts/chainlink-cluster/templates/ccip-scripts-job.yaml | 5 +++++ charts/chainlink-cluster/values.yaml | 6 ++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml index 36475e68e2..d61a031a1b 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml @@ -10,8 +10,7 @@ data: "Env": "{{$.Release.Namespace}}", "Bootstrap": { {{- with (index $.Values.chainlink.nodes 0) }} - {{- $nameWithoutHyphen := .name | replace "-" "" }} - "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", + "URL": "http://app-{{.name}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.chainlink.web_port}}", "Email": "notreal@fakeemail.ch", "Password": "fj293fbBnlQ!f9vNs", "InternalIP": "{{$.Release.Name}}-{{.name}}", @@ -23,7 +22,7 @@ data: {{- if ne $index 0}} {{- $nameWithoutHyphen := $cfg.name | replace "-" "" }} {{- if ne $index 1 }},{{- end }}{ - "URL": "https://{{$.Release.Namespace}}-{{$nameWithoutHyphen}}.{{$.Values.ingress.baseDomain}}/", + "URL": "http://app-{{$cfg.name}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.chainlink.web_port}}", "Email": "notreal@fakeemail.ch", "Password": "fj293fbBnlQ!f9vNs", "InternalIP": "{{$.Release.Name}}-{{$cfg.name}}", @@ -44,7 +43,7 @@ data: "CCIPChains": { {{- range $index, $cfg := $.Values.ccip.chains }} {{- if ne $index 0 }},{{- end }}"{{$cfg.ChainId}}":{ - "NetworkURL": "wss://{{$.Release.Namespace}}-geth-{{$cfg.NetworkId}}-ws.{{$.Values.ingress.baseDomain}}", + "NetworkURL": "ws://geth-{{$cfg.NetworkId}}.{{$.Release.Namespace}}.svc.cluster.local:{{.Values.geth.wsrpcPort}}", "WalletKey": {{$cfg.WalletKey | quote}}, "DeployLink": {{$cfg.DeployLink}}, "DeployWETH": {{$cfg.DeployWETH}}, diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index 156eb0c3fe..1dd8e12745 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -8,6 +8,11 @@ metadata: prometheus.io/scrape: 'true' spec: template: + metadata: + labels: + app: ccip-scripts-deployer + annotations: + prometheus.io/scrape: 'true' spec: containers: - name: ccip-scripts diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index 9d7980db0c..24db09dae4 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -120,8 +120,8 @@ geth: runAsUser: 999 runAsGroup: 999 version: v1.12.0 - wsrpc-port: 8546 - httprpc-port: 8544 + wsrpcPort: 8546 + httprpcPort: 8544 blocktime: 1 chains: - networkId: 1337 @@ -208,8 +208,6 @@ ccip: geth_1337: geth_2337: - - # mockserver is https://www.mock-server.com/where/kubernetes.html # used to stub External Adapters mockserver: From dd145dd78fa2070364e83d932c449f7a555a02aa Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Wed, 20 Mar 2024 16:33:20 +0100 Subject: [PATCH 07/13] fix wsRpc props and use latest docker image --- charts/chainlink-cluster/devspace.yaml | 4 ++-- charts/chainlink-cluster/templates/ccip-scripts-cm.yaml | 2 +- charts/chainlink-cluster/templates/ccip-scripts-job.yaml | 2 +- charts/chainlink-cluster/values.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 5df107ffed..c41b174171 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -520,8 +520,8 @@ deployments: runAsUser: 999 runAsGroup: 999 version: v1.12.0 - wsrpc-port: 8546 - httprpc-port: 8544 + wsRpcPort: 8546 + httpRpcPort: 8544 chains: - networkId: 1337 customEVMConfigToml: | diff --git a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml index d61a031a1b..8161a97885 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml @@ -43,7 +43,7 @@ data: "CCIPChains": { {{- range $index, $cfg := $.Values.ccip.chains }} {{- if ne $index 0 }},{{- end }}"{{$cfg.ChainId}}":{ - "NetworkURL": "ws://geth-{{$cfg.NetworkId}}.{{$.Release.Namespace}}.svc.cluster.local:{{.Values.geth.wsrpcPort}}", + "NetworkURL": "ws://geth-{{$cfg.NetworkId}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.geth.wsRpcPort}}", "WalletKey": {{$cfg.WalletKey | quote}}, "DeployLink": {{$cfg.DeployLink}}, "DeployWETH": {{$cfg.DeployWETH}}, diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index 1dd8e12745..a6ec73c950 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -20,7 +20,7 @@ spec: env: - name: CONFIG_JSON_PATH value: /data/config.json - command: [ "go", "test", "pilot/crib_test.go", "-v" ] + command: [ "pilot.test", "-test.v" ] volumeMounts: - name: config-volume mountPath: /data diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index 24db09dae4..6ca49be9c8 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -120,8 +120,8 @@ geth: runAsUser: 999 runAsGroup: 999 version: v1.12.0 - wsrpcPort: 8546 - httprpcPort: 8544 + wsRpcPort: 8546 + httpRpcPort: 8544 blocktime: 1 chains: - networkId: 1337 From bf447463c2433649ca5480cf76939e6441bb9319 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Thu, 21 Mar 2024 09:51:59 +0100 Subject: [PATCH 08/13] wait for endpoints init container --- .../templates/ccip-scripts-job.yaml | 25 ++++++++++++- .../templates/ccip-scripts-scripts-cm.yaml | 36 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index a6ec73c950..d5320bdcd1 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -14,13 +14,32 @@ spec: annotations: prometheus.io/scrape: 'true' spec: + initContainers: + - name: wait-for-nodes + image: {{.Values.ccip.ccipScriptsImage}} + volumeMounts: + - name: scripts-volume + mountPath: /scripts + securityContext: + capabilities: + drop: + - ALL + runAsUser: 999 + runAsGroup: 999 + runAsNonRoot: true + command: + - bash + - /scripts/wait_for_endpoint.sh + {{- with (index $.Values.chainlink.nodes 0) }} + - "http://app-{{.name}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.chainlink.web_port}}" + {{- end }} containers: - name: ccip-scripts image: {{.Values.ccip.ccipScriptsImage}} env: - name: CONFIG_JSON_PATH value: /data/config.json - command: [ "pilot.test", "-test.v" ] + command: [ "pilot.test", "-test.v" ] volumeMounts: - name: config-volume mountPath: /data @@ -36,4 +55,8 @@ spec: - name: config-volume configMap: name: ccip-scripts-config + - name: scripts-volume + configMap: + defaultMode: 0755 + name: ccip-scripts-scripts backoffLimit: 1 diff --git a/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml new file mode 100644 index 0000000000..43082a6ee6 --- /dev/null +++ b/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ccip-scripts-scripts +data: + wait_for_endpoint.sh: | + #!/bin/bash + + # Define your endpoint URL + ENDPOINT_URL="${1}" + + if [ "${ENDPOINT_URL}" = "" ]; then + echo "Need an argument for the ENDPOINT_URL in argument 1" + exit 1 + fi + + # Function to check if the endpoint returns a 200 status code + check_endpoint() { + status_code=$(curl -s -o /dev/null -w "%{http_code}" "$ENDPOINT_URL") + if [ "$status_code" == "200" ]; then + echo "Endpoint is ready!" + exit 0 + else + echo "Endpoint is not yet ready, status code: $status_code" + return 1 + fi + } + + # Main loop to periodically check the endpoint + echo "Waiting for endpoint to be ready..." + while ! check_endpoint; do + sleep 5 + done + + echo "Endpoint is ready!" + From c99936d3be7ea917cbed4adaf62d5ea9e84a07a7 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Thu, 21 Mar 2024 12:52:08 +0100 Subject: [PATCH 09/13] run ccip-scripts job as helm post install hook --- charts/chainlink-cluster/README.md | 28 +++++++++++++------ charts/chainlink-cluster/devspace.yaml | 1 + .../templates/ccip-scripts-job.yaml | 9 +++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 8d7044ceb4..56133e0994 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -14,22 +14,28 @@ nix develop # Develop -## New cluster +## Deploying New cluster We are using [devspace](https://www.devspace.sh/docs/getting-started/installation?x0=3) -Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details +1) Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details -Set up your K8s access -``` -export DEVSPACE_IMAGE="..." -./setup.sh ${my-personal-namespace-name-crib} -``` +2) Set up env variables required in devspace.yaml: + ``` + export DEVSPACE_IMAGE=... + export DEVSPACE_INGRESS_CIDRS="0.0.0.0/0" + export DEVSPACE_INGRESS_BASE_DOMAIN=... + export DEVSPACE_INGRESS_CERT_ARN=... + export DEVSPACE_CCIP_SCRIPTS_IMAGE=... + ``` +3) Configure access to your kubernetes cluster -Build and deploy current commit +4) Build and deploy current commit ``` devspace deploy ``` +### Additional Configuration options + Default `ttl` is `72h`, use `ttl` command to update if you need more time Valid values are `1h`, `2m`, `3s`, etc. Go time format is invalid `1h2m3s` @@ -64,6 +70,12 @@ Destroy the cluster devspace purge ``` +## CCIP Contracts and Jobs Deployment +By default, the helm chart includes a post install hook defined in the ccip-scripts-deploy job. +It will deploy contracts and jobs to make the CCIP enabled cluster operational. + +`ccip-scripts-deploy` job usually takes around 6 minutes to complete. + ## Running load tests Check this [doc](../../integration-tests/load/ocr/README.md) diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index c41b174171..73adfb7d3f 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -139,6 +139,7 @@ deployments: namespace: ${DEVSPACE_NAMESPACE} helm: releaseName: "app" + upgradeArgs: ["--timeout", "10m"] chart: name: cl-cluster path: . diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index d5320bdcd1..3ca5591270 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -4,13 +4,20 @@ metadata: name: ccip-scripts-deploy labels: app: ccip-scripts-deployer + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" annotations: prometheus.io/scrape: 'true' + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-install + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation spec: template: metadata: labels: app: ccip-scripts-deployer + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" annotations: prometheus.io/scrape: 'true' spec: @@ -59,4 +66,4 @@ spec: configMap: defaultMode: 0755 name: ccip-scripts-scripts - backoffLimit: 1 + backoffLimit: 0 From b05115c42668430ac98255271d1b8b7463921f82 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Thu, 21 Mar 2024 13:24:03 +0100 Subject: [PATCH 10/13] make scripts deploy and optional feature --- charts/chainlink-cluster/devspace.yaml | 9 ++++++--- charts/chainlink-cluster/templates/ccip-scripts-cm.yaml | 2 ++ charts/chainlink-cluster/templates/ccip-scripts-job.yaml | 4 +++- .../templates/ccip-scripts-scripts-cm.yaml | 2 ++ charts/chainlink-cluster/values.yaml | 1 + 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 73adfb7d3f..cb4d5e752e 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -74,8 +74,7 @@ pipelines: --set=helm.values.chainlink.nodes[3].image=$image \ --set=helm.values.chainlink.nodes[4].image=$image \ --set=helm.values.chainlink.nodes[5].image=$image \ - --set=helm.values.ingress.baseDomain=$DEVSPACE_INGRESS_BASE_DOMAIN \ - --set=helm.values.ccip.ccipScriptsImage=$DEVSPACE_CCIP_SCRIPTS_IMAGE + --set=helm.values.ingress.baseDomain=$DEVSPACE_INGRESS_BASE_DOMAIN echo echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}" echo "To extend the TTL for e.g. 72 hours, run: devspace run ttl ${DEVSPACE_NAMESPACE} 72h" @@ -139,7 +138,8 @@ deployments: namespace: ${DEVSPACE_NAMESPACE} helm: releaseName: "app" - upgradeArgs: ["--timeout", "10m"] + upgradeArgs: ["--timeout", "10m", "--debug"] + displayOutput: true chart: name: cl-cluster path: . @@ -544,6 +544,9 @@ deployments: limits: cpu: 1 memory: 1024Mi + ccip: + deployContractsAndJobs: true + ccipScriptsImage: ${DEVSPACE_CCIP_SCRIPTS_IMAGE} # mockserver is https://www.mock-server.com/where/kubernetes.html # used to stub External Adapters mockserver: diff --git a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml index 8161a97885..628c6ba321 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml @@ -1,3 +1,4 @@ +{{- if $.Values.ccip.deployContractsAndJobs }} apiVersion: v1 kind: ConfigMap metadata: @@ -53,3 +54,4 @@ data: }, "LaneDeploySettings": {{$.Values.ccip.LaneDeploySettings | toJson }} } +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index 3ca5591270..8a09dfdbfa 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -1,3 +1,4 @@ +{{- if $.Values.ccip.deployContractsAndJobs }} apiVersion: batch/v1 kind: Job metadata: @@ -9,7 +10,7 @@ metadata: prometheus.io/scrape: 'true' # This is what defines this resource as a hook. Without this line, the # job is considered part of the release. - "helm.sh/hook": post-install + "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-weight": "0" "helm.sh/hook-delete-policy": before-hook-creation spec: @@ -67,3 +68,4 @@ spec: defaultMode: 0755 name: ccip-scripts-scripts backoffLimit: 0 +{{- end}} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml index 43082a6ee6..abda92d3ea 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml @@ -1,3 +1,4 @@ +{{- if $.Values.ccip.deployContractsAndJobs }} apiVersion: v1 kind: ConfigMap metadata: @@ -34,3 +35,4 @@ data: echo "Endpoint is ready!" +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index 6ca49be9c8..e4e69257b3 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -145,6 +145,7 @@ geth: ccip: ccipScriptsImage: ccip-scripts-image:image-tag + deployContractsAndJobs: false chains: - NetworkId: 1337 ChainId: geth_1337 From 826309bb2776ba02077f3dccfcc20b5f77a37415 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Thu, 21 Mar 2024 13:30:24 +0100 Subject: [PATCH 11/13] pass env var in the values directly --- charts/chainlink-cluster/devspace.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index cb4d5e752e..36b469a2da 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -73,8 +73,8 @@ pipelines: --set=helm.values.chainlink.nodes[2].image=$image \ --set=helm.values.chainlink.nodes[3].image=$image \ --set=helm.values.chainlink.nodes[4].image=$image \ - --set=helm.values.chainlink.nodes[5].image=$image \ - --set=helm.values.ingress.baseDomain=$DEVSPACE_INGRESS_BASE_DOMAIN + --set=helm.values.chainlink.nodes[5].image=$image + echo echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}" echo "To extend the TTL for e.g. 72 hours, run: devspace run ttl ${DEVSPACE_NAMESPACE} 72h" @@ -138,7 +138,7 @@ deployments: namespace: ${DEVSPACE_NAMESPACE} helm: releaseName: "app" - upgradeArgs: ["--timeout", "10m", "--debug"] + upgradeArgs: ["--timeout", "10m"] displayOutput: true chart: name: cl-cluster @@ -580,6 +580,7 @@ deployments: # These ingresses create AWS ALB resources and Route 53 Records. ingress: enabled: true + baseDomain: ${DEVSPACE_INGRESS_BASE_DOMAIN} annotation_certificate_arn: ${DEVSPACE_INGRESS_CERT_ARN} annotation_group_name: ${DEVSPACE_NAMESPACE} hosts: From 8c6902aab1d821d372f552602dbc0f25e9235072 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Fri, 22 Mar 2024 09:48:40 +0100 Subject: [PATCH 12/13] Apply suggestions from code review Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> --- charts/chainlink-cluster/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 56133e0994..9879a6e1d9 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -122,7 +122,7 @@ Install ``` helm install -f values.yaml cl-cluster . \ --set=ingress.baseDomain="$DEVSPACE_INGRESS_BASE_DOMAIN" \ - --set=ccip.ccipScriptsImage=$DEVSPACE_CCIP_SCRIPTS_IMAGE + --set=ccip.ccipScriptsImage="$DEVSPACE_CCIP_SCRIPTS_IMAGE" ``` From 30f46b19c54f218c884a02f91e99c2a68c7ce876 Mon Sep 17 00:00:00 2001 From: Radek Scheibinger Date: Fri, 22 Mar 2024 10:15:21 +0100 Subject: [PATCH 13/13] suggestions from PR comments --- charts/chainlink-cluster/templates/ccip-scripts-job.yaml | 2 +- .../chainlink-cluster/templates/mockserver-networkpolicy.yaml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml index 8a09dfdbfa..80303fbce7 100644 --- a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: ccip-scripts-deploy + name: ccip-scripts-deployer labels: app: ccip-scripts-deployer helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" diff --git a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml index 2efa7d2ed8..8d167b4f92 100644 --- a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml @@ -15,10 +15,6 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} - # Allow traffic from ccip-scripts-deployer - - podSelector: - matchLabels: - app: ccip-scripts-deployer ports: - protocol: TCP port: 1080