diff --git a/charts/chainlink-cluster/.gitignore b/charts/chainlink-cluster/.gitignore index 3ee791f740..0b1ce86a15 100644 --- a/charts/chainlink-cluster/.gitignore +++ b/charts/chainlink-cluster/.gitignore @@ -1,3 +1,4 @@ # Helm charts/ requirements.lock +.rendered \ No newline at end of file diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index b0f0966857..86452cc872 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -14,22 +14,28 @@ nix develop # Develop -## New cluster +## Deploying New cluster We are using [devspace](https://www.devspace.sh/docs/getting-started/installation?x0=3) -Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details +1) Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments for more details -Set up your K8s access -``` -export DEVSPACE_IMAGE="..." -./setup.sh ${my-personal-namespace-name-crib} -``` +2) Set up env variables required in devspace.yaml: + ``` + export DEVSPACE_IMAGE=... + export DEVSPACE_INGRESS_CIDRS="0.0.0.0/0" + export DEVSPACE_INGRESS_BASE_DOMAIN=... + export DEVSPACE_INGRESS_CERT_ARN=... + export DEVSPACE_CCIP_SCRIPTS_IMAGE=... + ``` +3) Configure access to your kubernetes cluster -Build and deploy current commit +4) Build and deploy current commit ``` devspace deploy ``` +### Additional Configuration options + Default `ttl` is `72h`, use `ttl` command to update if you need more time Valid values are `1h`, `2m`, `3s`, etc. Go time format is invalid `1h2m3s` @@ -64,6 +70,12 @@ Destroy the cluster devspace purge ``` +## CCIP Contracts and Jobs Deployment +By default, the helm chart includes a post install hook defined in the ccip-scripts-deploy job. +It will deploy contracts and jobs to make the CCIP enabled cluster operational. + +`ccip-scripts-deploy` job usually takes around 6 minutes to complete. + ## Running load tests Check this [doc](../../integration-tests/load/ocr/README.md) @@ -108,9 +120,12 @@ kubectl config set-context --current --namespace cl-cluster Install ``` -helm install -f values.yaml cl-cluster . +helm install -f values.yaml cl-cluster . \ + --set=ingress.baseDomain="$DEVSPACE_INGRESS_BASE_DOMAIN" \ + --set=ccip.ccipScriptsImage="$DEVSPACE_CCIP_SCRIPTS_IMAGE" ``` + ## Create a new release Bump version in `Chart.yml` add your changes and add `helm_release` label to any PR to trigger a release diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index e7cf75d493..36b469a2da 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -17,6 +17,9 @@ vars: source: env # Time to wait for pods to be in `Ready` condition DEVSPACE_K8S_POD_WAIT_TIMEOUT: 600s + # Image URI required for deploying CCIP Contracts and Jobs + DEVSPACE_CCIP_SCRIPTS_IMAGE: + source: env # This is a list of `pipelines` that DevSpace can execute (you can define your own) pipelines: @@ -71,6 +74,7 @@ pipelines: --set=helm.values.chainlink.nodes[3].image=$image \ --set=helm.values.chainlink.nodes[4].image=$image \ --set=helm.values.chainlink.nodes[5].image=$image + echo echo "Namespace ${DEVSPACE_NAMESPACE} will be deleted in ${NS_TTL}" echo "To extend the TTL for e.g. 72 hours, run: devspace run ttl ${DEVSPACE_NAMESPACE} 72h" @@ -134,6 +138,8 @@ deployments: namespace: ${DEVSPACE_NAMESPACE} helm: releaseName: "app" + upgradeArgs: ["--timeout", "10m"] + displayOutput: true chart: name: cl-cluster path: . @@ -515,8 +521,8 @@ deployments: runAsUser: 999 runAsGroup: 999 version: v1.12.0 - wsrpc-port: 8546 - httprpc-port: 8544 + wsRpcPort: 8546 + httpRpcPort: 8544 chains: - networkId: 1337 customEVMConfigToml: | @@ -538,6 +544,9 @@ deployments: limits: cpu: 1 memory: 1024Mi + ccip: + deployContractsAndJobs: true + ccipScriptsImage: ${DEVSPACE_CCIP_SCRIPTS_IMAGE} # mockserver is https://www.mock-server.com/where/kubernetes.html # used to stub External Adapters mockserver: @@ -571,6 +580,7 @@ deployments: # These ingresses create AWS ALB resources and Route 53 Records. ingress: enabled: true + baseDomain: ${DEVSPACE_INGRESS_BASE_DOMAIN} annotation_certificate_arn: ${DEVSPACE_INGRESS_CERT_ARN} annotation_group_name: ${DEVSPACE_NAMESPACE} hosts: diff --git a/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml new file mode 100644 index 0000000000..628c6ba321 --- /dev/null +++ b/charts/chainlink-cluster/templates/ccip-scripts-cm.yaml @@ -0,0 +1,57 @@ +{{- if $.Values.ccip.deployContractsAndJobs }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: ccip-scripts-config +data: + config.json: | + { + "EnvName": "{{$.Release.Namespace}}", + "DONCreds": { + "Env": "{{$.Release.Namespace}}", + "Bootstrap": { + {{- with (index $.Values.chainlink.nodes 0) }} + "URL": "http://app-{{.name}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.chainlink.web_port}}", + "Email": "notreal@fakeemail.ch", + "Password": "fj293fbBnlQ!f9vNs", + "InternalIP": "{{$.Release.Name}}-{{.name}}", + {{- end}} + "HTTPTimeout": null + }, + "Nodes": [ + {{- range $index, $cfg := $.Values.chainlink.nodes }} + {{- if ne $index 0}} + {{- $nameWithoutHyphen := $cfg.name | replace "-" "" }} + {{- if ne $index 1 }},{{- end }}{ + "URL": "http://app-{{$cfg.name}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.chainlink.web_port}}", + "Email": "notreal@fakeemail.ch", + "Password": "fj293fbBnlQ!f9vNs", + "InternalIP": "{{$.Release.Name}}-{{$cfg.name}}", + "HTTPTimeout": null + } + {{- end}} + {{- end}} + ] + }, + {{- $networkIDs := list }} + {{- range $index, $cfg :=$.Values.ccip.chains }} + {{- $networkIDs = append $networkIDs $cfg.ChainId }} + {{- end }} + + {{- $delimiter := "," }} + {{- $foldedString := join $delimiter $networkIDs }} + "ChainPairs":[{{- printf $foldedString | quote }}], + "CCIPChains": { + {{- range $index, $cfg := $.Values.ccip.chains }} + {{- if ne $index 0 }},{{- end }}"{{$cfg.ChainId}}":{ + "NetworkURL": "ws://geth-{{$cfg.NetworkId}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.geth.wsRpcPort}}", + "WalletKey": {{$cfg.WalletKey | quote}}, + "DeployLink": {{$cfg.DeployLink}}, + "DeployWETH": {{$cfg.DeployWETH}}, + "ChainConfig": {{$cfg.ChainConfig | toJson}} + } + {{- end}} + }, + "LaneDeploySettings": {{$.Values.ccip.LaneDeploySettings | toJson }} + } +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/ccip-scripts-job.yaml b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml new file mode 100644 index 0000000000..80303fbce7 --- /dev/null +++ b/charts/chainlink-cluster/templates/ccip-scripts-job.yaml @@ -0,0 +1,71 @@ +{{- if $.Values.ccip.deployContractsAndJobs }} +apiVersion: batch/v1 +kind: Job +metadata: + name: ccip-scripts-deployer + labels: + app: ccip-scripts-deployer + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + prometheus.io/scrape: 'true' + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + metadata: + labels: + app: ccip-scripts-deployer + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + prometheus.io/scrape: 'true' + spec: + initContainers: + - name: wait-for-nodes + image: {{.Values.ccip.ccipScriptsImage}} + volumeMounts: + - name: scripts-volume + mountPath: /scripts + securityContext: + capabilities: + drop: + - ALL + runAsUser: 999 + runAsGroup: 999 + runAsNonRoot: true + command: + - bash + - /scripts/wait_for_endpoint.sh + {{- with (index $.Values.chainlink.nodes 0) }} + - "http://app-{{.name}}.{{$.Release.Namespace}}.svc.cluster.local:{{$.Values.chainlink.web_port}}" + {{- end }} + containers: + - name: ccip-scripts + image: {{.Values.ccip.ccipScriptsImage}} + env: + - name: CONFIG_JSON_PATH + value: /data/config.json + command: [ "pilot.test", "-test.v" ] + volumeMounts: + - name: config-volume + mountPath: /data + securityContext: + capabilities: + drop: + - ALL + runAsUser: 999 + runAsGroup: 999 + runAsNonRoot: true + restartPolicy: Never + volumes: + - name: config-volume + configMap: + name: ccip-scripts-config + - name: scripts-volume + configMap: + defaultMode: 0755 + name: ccip-scripts-scripts + backoffLimit: 0 +{{- end}} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml b/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml new file mode 100644 index 0000000000..abda92d3ea --- /dev/null +++ b/charts/chainlink-cluster/templates/ccip-scripts-scripts-cm.yaml @@ -0,0 +1,38 @@ +{{- if $.Values.ccip.deployContractsAndJobs }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: ccip-scripts-scripts +data: + wait_for_endpoint.sh: | + #!/bin/bash + + # Define your endpoint URL + ENDPOINT_URL="${1}" + + if [ "${ENDPOINT_URL}" = "" ]; then + echo "Need an argument for the ENDPOINT_URL in argument 1" + exit 1 + fi + + # Function to check if the endpoint returns a 200 status code + check_endpoint() { + status_code=$(curl -s -o /dev/null -w "%{http_code}" "$ENDPOINT_URL") + if [ "$status_code" == "200" ]; then + echo "Endpoint is ready!" + exit 0 + else + echo "Endpoint is not yet ready, status code: $status_code" + return 1 + fi + } + + # Main loop to periodically check the endpoint + echo "Waiting for endpoint to be ready..." + while ! check_endpoint; do + sleep 5 + done + + echo "Endpoint is ready!" + +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml index 5f7e7706ce..3e4c9f49b4 100644 --- a/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/chainlink-db-networkpolicy.yaml @@ -15,10 +15,6 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} - # Allow all runner pods to access the database pods. - - podSelector: - matchLabels: - app: runner ports: - protocol: TCP port: 5432 diff --git a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml index e63759a994..fe6df3cf0f 100644 --- a/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/chainlink-node-networkpolicy.yaml @@ -15,7 +15,8 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} + # Allow traffic from ccip-scripts-deployer - podSelector: matchLabels: - app: runner + app: ccip-scripts-deployer {{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml index 025d618450..38d0f07445 100644 --- a/charts/chainlink-cluster/templates/geth-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/geth-networkpolicy.yaml @@ -15,10 +15,10 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} - # Allow http and websocket connections from the runner pods. + # Allow traffic from ccip-scripts-deployer - podSelector: matchLabels: - app: runner + app: ccip-scripts-deployer ports: - protocol: TCP port: 8544 diff --git a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml index 6ac4f658e3..8d167b4f92 100644 --- a/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml +++ b/charts/chainlink-cluster/templates/mockserver-networkpolicy.yaml @@ -15,10 +15,6 @@ spec: - podSelector: matchLabels: app: {{ $.Release.Name }} - # Allow http traffic from the runner pods. - - podSelector: - matchLabels: - app: runner ports: - protocol: TCP port: 1080 diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml index b0866574c9..e4e69257b3 100644 --- a/charts/chainlink-cluster/values.yaml +++ b/charts/chainlink-cluster/values.yaml @@ -120,8 +120,8 @@ geth: runAsUser: 999 runAsGroup: 999 version: v1.12.0 - wsrpc-port: 8546 - httprpc-port: 8544 + wsRpcPort: 8546 + httpRpcPort: 8544 blocktime: 1 chains: - networkId: 1337 @@ -142,6 +142,73 @@ geth: limits: cpu: 1 memory: 1024Mi + +ccip: + ccipScriptsImage: ccip-scripts-image:image-tag + deployContractsAndJobs: false + chains: + - NetworkId: 1337 + ChainId: geth_1337 + WalletKey: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + DeployLink: true + DeployWETH: true + ChainConfig: + EvmChainId: 1337 + GasSettings: + EIP1559: true + GasTipCap: 1000000000 + FeeTokens: + - "ChainLink Token" + - "WETH" + WrappedNative: "WETH" + Router: "0x0000000000000000000000000000000000000000" + ARM: "0x0000000000000000000000000000000000000000" + ARMProxy: "0x0000000000000000000000000000000000000000" + PriceRegistry: "0x0000000000000000000000000000000000000000" + DeploySettings: + DeployARM: true + DeployTokenPools: true + DeployRouter: true + DeployPriceRegistry: true + TunableChainValues: + FinalityDepth: 1 + OptimisticConfirmations: 1 + MaxGasPrice: 200000000000 + - NetworkId: 2337 + ChainId: geth_2337 + WalletKey: "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d" + DeployLink: true + DeployWETH: true + ChainConfig: + EvmChainId: 2337 + GasSettings: + EIP1559: true + GasTipCap: 1000000000 + FeeTokens: + - "ChainLink Token" + - "WETH" + WrappedNative: "WETH" + Router: "0x0000000000000000000000000000000000000000" + ARM: "0x0000000000000000000000000000000000000000" + ARMProxy: "0x0000000000000000000000000000000000000000" + PriceRegistry: "0x0000000000000000000000000000000000000000" + DeploySettings: + DeployARM: true + DeployTokenPools: true + DeployRouter: true + DeployPriceRegistry: true + TunableChainValues: + FinalityDepth: 1 + OptimisticConfirmations: 1 + MaxGasPrice: 200000000000 + LaneDeploySettings: + geth_1337,geth_2337: + DeployLane: true + DeployPingPongDapp: true + CCIPLaneConfigs: + geth_1337: + geth_2337: + # mockserver is https://www.mock-server.com/where/kubernetes.html # used to stub External Adapters mockserver: @@ -270,6 +337,7 @@ ingress: enabled: false annotations: {} ingressClassName: alb + baseDomain: "example.com" hosts: - host: chainlink-node-1.local http: