From 26bf6da954101502003160522a732216cda47c61 Mon Sep 17 00:00:00 2001 From: chainchad <96362174+chainchad@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:10:39 -0500 Subject: [PATCH] Prep for CRIB (#11418) * Make CICD work with new ECRs (#495) * Fix ECR and publish on PRs (#496) * Remove test file * Add mockserver to helm chart dependency * Add default values file * Add security contexts * Fix ref to values * Use head SHA instead of commit SHA for image tagging * Use emptyDir mount for postgres to resolve perm issues * Fix volume mounts on postgres * Fix /tmp writes and mount /tmp emptyDir * Remove custom uid * Run as same uid as postgres * Test DOCKER_METADATA_PR_HEAD_SHA disable for root image * Upgrade docker metadata action * Do a full clone * Avoid emptyDir mounts and update gid * Use head SHA when on a PR as default input * Downgrade metata action to latest on prev major * Override sha tag with HEAD value * Remove old sha tag default * Attempt to fix sha tag * Fix syntax error * Remove DOCKER_METADATA_PR_HEAD_SHA env * Refactor shared var output * Revert "Refactor shared var output" This reverts commit 2362fe6473974d4723cf5294cccec7090ab07a0f. * Revert "Remove DOCKER_METADATA_PR_HEAD_SHA env" This reverts commit a7bd01e544ad97603147ccb33576e8006056e930. * Try to make things work again * Set the host to localhost for K8s * Add user to pg_isready * Make securityContext container specific * Create init container to create /clroot * Fix volumes * Add security context to init container * Skip the chown * Remove init container * Breakout DB into its own deployment and service * Fix typo * Update localhost to db service dns * Fix path to script * Change geth /root path to /app * Create emptyDir volume for geth devchain dir * Remove full clone * Add correct inputs to mockserver * merge with develop * cleanup & verify * Add fixes for /chainlink * Remove unused env var * Set back to hardcoded repo name * Reset values back to pre rebase changes * Remove mockserver templates in favor of subchart * fix connect.toml and rename mockserver connection * Backout triggering on PR * Create new build-publish workflow for chainlink-untrusted (from PR) * Reset workflow back * Make step name accurate --------- Co-authored-by: skudasov --- .../build-sign-publish-chainlink/action.yml | 12 +- .../workflows/automation-ondemand-tests.yml | 2 +- .github/workflows/build-publish-pr.yml | 44 +++++ charts/chainlink-cluster/Chart.yaml | 7 +- charts/chainlink-cluster/README.md | 6 +- charts/chainlink-cluster/connect.toml | 2 +- .../chainlink-cluster/dashboard/dashboard.go | 42 ---- charts/chainlink-cluster/devspace.yaml | 180 ++++++++++++++++-- ...ment.yaml => chainlink-db-deployment.yaml} | 72 ++----- .../templates/chainlink-db-service.yaml | 16 ++ .../templates/chainlink-node-deployment.yaml | 97 ++++++++++ ...rvice.yaml => chainlink-node-service.yaml} | 0 .../templates/geth-config-map.yaml | 6 +- .../templates/geth-deployment.yaml | 24 ++- .../templates/mockserver-service.yaml | 14 -- .../templates/mockserver.yaml | 61 ------ .../templates/runner-deployment.yaml | 4 + charts/chainlink-cluster/values-raw-helm.yaml | 134 ------------- charts/chainlink-cluster/values.yaml | 178 +++++++++++++++++ core/chainlink.devspace.Dockerfile | 2 +- 20 files changed, 554 insertions(+), 349 deletions(-) create mode 100644 .github/workflows/build-publish-pr.yml rename charts/chainlink-cluster/templates/{chainlink-deployment.yaml => chainlink-db-deployment.yaml} (60%) create mode 100644 charts/chainlink-cluster/templates/chainlink-db-service.yaml create mode 100644 charts/chainlink-cluster/templates/chainlink-node-deployment.yaml rename charts/chainlink-cluster/templates/{chainlink-service.yaml => chainlink-node-service.yaml} (100%) delete mode 100644 charts/chainlink-cluster/templates/mockserver-service.yaml delete mode 100755 charts/chainlink-cluster/templates/mockserver.yaml delete mode 100644 charts/chainlink-cluster/values-raw-helm.yaml create mode 100644 charts/chainlink-cluster/values.yaml diff --git a/.github/actions/build-sign-publish-chainlink/action.yml b/.github/actions/build-sign-publish-chainlink/action.yml index 62add53092a..8c79f651afd 100644 --- a/.github/actions/build-sign-publish-chainlink/action.yml +++ b/.github/actions/build-sign-publish-chainlink/action.yml @@ -34,7 +34,7 @@ inputs: required: false git-commit-sha: description: Git commit SHA used as metadata when building the application (appears in logs) - default: ${{ github.sha }} + default: ${{ github.event.pull_request.head.sha || github.sha }} required: false aws-role-to-assume: description: The AWS role to assume as the CD user, if any. Used in configuring the docker/login-action @@ -73,7 +73,7 @@ runs: using: composite steps: - name: Set shared variables - shell: sh + shell: bash # See https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings run: | SHARED_IMAGES=${{ inputs.ecr-hostname }}/${{ inputs.ecr-image-name }} @@ -122,7 +122,9 @@ runs: - name: Generate docker metadata for root image id: meta-root - uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v4.4.4 + uses: docker/metadata-action@2c0bd771b40637d97bf205cbccdd294a32112176 # v4.5.0 + env: + DOCKER_METADATA_PR_HEAD_SHA: "true" with: # list of Docker images to use as base name for tags images: ${{ env.shared-images }} @@ -164,7 +166,9 @@ runs: - name: Generate docker metadata for non-root image id: meta-nonroot - uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v4.4.4 + uses: docker/metadata-action@2c0bd771b40637d97bf205cbccdd294a32112176 # v4.5.0 + env: + DOCKER_METADATA_PR_HEAD_SHA: "true" with: flavor: | latest=auto diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml index 5cd2182ff62..016a10252be 100644 --- a/.github/workflows/automation-ondemand-tests.yml +++ b/.github/workflows/automation-ondemand-tests.yml @@ -115,7 +115,7 @@ jobs: pull-requests: write id-token: write contents: read - needs: [ build-chainlink, build-test-image ] + needs: [build-chainlink, build-test-image] env: CHAINLINK_COMMIT_SHA: ${{ github.sha }} CHAINLINK_ENV_USER: ${{ github.actor }} diff --git a/.github/workflows/build-publish-pr.yml b/.github/workflows/build-publish-pr.yml new file mode 100644 index 00000000000..b958295cf24 --- /dev/null +++ b/.github/workflows/build-publish-pr.yml @@ -0,0 +1,44 @@ +name: "Build and Publish from PR" + +## +# This workflow builds and publishes a Docker image for Chainlink from a PR. +# It doesn't use an environment, has its own special IAM role, does not sign +# the image, and publishes to a special ECR repo. +## + +on: + pull_request: + +jobs: + build-publish-untrusted: + if: ${{ ! startsWith(github.ref_name, 'release/') }} + runs-on: ubuntu-20.04 + permissions: + id-token: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Build and publish chainlink image + uses: ./.github/actions/build-sign-publish-chainlink + with: + publish: true + aws-role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_PUBLISH_PR_ARN }} + aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS_DEFAULT }} + aws-region: ${{ secrets.AWS_REGION }} + sign-images: false + ecr-hostname: ${{ secrets.AWS_SDLC_ECR_HOSTNAME }} + ecr-image-name: chainlink-untrusted + dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} + dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} + + - name: Collect Metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: build-publish-untrusted + continue-on-error: true diff --git a/charts/chainlink-cluster/Chart.yaml b/charts/chainlink-cluster/Chart.yaml index bfea29c82ec..127f5b6e326 100644 --- a/charts/chainlink-cluster/Chart.yaml +++ b/charts/chainlink-cluster/Chart.yaml @@ -2,4 +2,9 @@ apiVersion: v1 name: chainlink-cluster description: Chainlink nodes cluster version: 0.1.3 -appVersion: '2.6.0' \ No newline at end of file +appVersion: "2.6.0" +dependencies: + - name: mockserver + version: "5.14.0" + repository: "@mockserver" + condition: mockserver.enabled \ No newline at end of file diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index e3cec129a91..5fb55536635 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -74,10 +74,10 @@ After that all the changes will be synced automatically Check `.profiles` to understand what is uploaded in profiles `runner` and `node` # Helm -If you would like to use `helm` directly, please uncomment data in `values-raw-helm.yaml` +If you would like to use `helm` directly, please uncomment data in `values.yaml` ## Install from local files ``` -helm install -f values-raw-helm.yaml cl-cluster . +helm install -f values.yaml cl-cluster . ``` Forward all apps (in another terminal) ``` @@ -99,7 +99,7 @@ kubectl config set-context --current --namespace cl-cluster Install ``` -helm install -f values-raw-helm.yaml cl-cluster chainlink-cluster/chainlink-cluster --version v0.1.2 +helm install -f values.yaml cl-cluster . ``` ## Create a new release diff --git a/charts/chainlink-cluster/connect.toml b/charts/chainlink-cluster/connect.toml index f0a74d4c144..1f49b5a6e37 100644 --- a/charts/chainlink-cluster/connect.toml +++ b/charts/chainlink-cluster/connect.toml @@ -9,4 +9,4 @@ cl_node_url_template = "http://app-node-%d:6688" cl_node_internal_dns_record_template = "app-node-%d" cl_node_user = "notreal@fakeemail.ch" cl_node_password = "fj293fbBnlQ!f9vNs" -mockserver_url = "http://app-mockserver:1080" \ No newline at end of file +mockserver_url = "http://mockserver:1080" \ No newline at end of file diff --git a/charts/chainlink-cluster/dashboard/dashboard.go b/charts/chainlink-cluster/dashboard/dashboard.go index b29140c0405..19a596b63e9 100644 --- a/charts/chainlink-cluster/dashboard/dashboard.go +++ b/charts/chainlink-cluster/dashboard/dashboard.go @@ -350,48 +350,6 @@ func (m *CLClusterDashboard) generate() error { ), ), ), - // logs - dashboard.Row( - "Logs", - row.Collapse(), - row.WithTimeSeries( - "Log Counters", - timeseries.Span(12), - timeseries.Height("200px"), - timeseries.DataSource(m.PrometheusDataSourceName), - timeseries.WithPrometheusTarget( - `log_panic_count{namespace="${namespace}"}`, - prometheus.Legend("{{pod}} - panic"), - ), - timeseries.WithPrometheusTarget( - `log_fatal_count{namespace="${namespace}"}`, - prometheus.Legend("{{pod}} - fatal"), - ), - timeseries.WithPrometheusTarget( - `log_critical_count{namespace="${namespace}"}`, - prometheus.Legend("{{pod}} - critical"), - ), - timeseries.WithPrometheusTarget( - `log_warn_count{namespace="${namespace}"}`, - prometheus.Legend("{{pod}} - warn"), - ), - timeseries.WithPrometheusTarget( - `log_error_count{namespace="${namespace}"}`, - prometheus.Legend("{{pod}} - error"), - ), - ), - m.logsRowOption("All errors", ` - {namespace="${namespace}", app="app", container="node"} - | json - | level="error" - | line_format "{{ .instance }} {{ .level }} {{ .ts }} {{ .logger }} {{ .caller }} {{ .msg }} {{ .version }} {{ .nodeTier }} {{ .nodeName }} {{ .node }} {{ .evmChainID }} {{ .nodeOrder }} {{ .mode }} {{ .nodeState }} {{ .sentryEventID }} {{ .stacktrace }}"`), - m.logsRowOption("Node 1", `{namespace="${namespace}", app="app", instance="node-1", container="node"}`), - m.logsRowOption("Node 2", `{namespace="${namespace}", app="app", instance="node-2", container="node"}`), - m.logsRowOption("Node 3", `{namespace="${namespace}", app="app", instance="node-3", container="node"}`), - m.logsRowOption("Node 4", `{namespace="${namespace}", app="app", instance="node-4", container="node"}`), - m.logsRowOption("Node 5", `{namespace="${namespace}", app="app", instance="node-5", container="node"}`), - m.logsRowOption("Node 6", `{namespace="${namespace}", app="app", instance="node-6", container="node"}`), - ), // HeadTracker dashboard.Row("Head tracker", row.Collapse(), diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 4f7cf8641a3..cb4c8bfce49 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -40,26 +40,56 @@ deployments: # they can be defined the same way in values.yml # devspace merging this "values" and "values.yml" before deploy values: - runner: - image: ${DEVSPACE_IMAGE} - stateful: false - geth: - version: v1.12.0 - wsrpc-port: 8546 - httprpc-port: 8544 - networkid: 1337 - blocktime: 1 - mockserver: - port: 1080 - db: - stateful: false + podSecurityContext: + fsGroup: 999 + chainlink: + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 14933 + runAsGroup: 999 web_port: 6688 p2p_port: 6690 nodes: - name: node-1 image: ${DEVSPACE_IMAGE} version: latest + # override default config per node + # for example, use OCRv2 P2P setup, the whole config + # toml: | + # RootDir = './clroot' + # [Log] + # JSONConsole = true + # Level = 'debug' + # [WebServer] + # AllowOrigins = '*' + # SecureCookies = false + # SessionTimeout = '999h0m0s' + # [OCR2] + # Enabled = true + # [P2P] + # [P2P.V2] + # Enabled = false + # AnnounceAddresses = [] + # DefaultBootstrappers = [] + # DeltaDial = '15s' + # DeltaReconcile = '1m0s' + # ListenAddresses = [] + # [[EVM]] + # ChainID = '1337' + # MinContractPayment = '0' + # [[EVM.Nodes]] + # Name = 'node-0' + # WSURL = 'ws://geth:8546' + # HTTPURL = 'http://geth:8544' + # [WebServer.TLS] + # HTTPSPort = 0 + # or use overridesToml to override some part of configuration + # overridesToml: | - name: node-2 image: ${DEVSPACE_IMAGE} version: latest @@ -75,11 +105,125 @@ deployments: - name: node-6 image: ${DEVSPACE_IMAGE} version: latest - prometheusMonitor: "true" - podAnnotations: { } - nodeSelector: { } - tolerations: [ ] - affinity: { } + resources: + requests: + cpu: 350m + memory: 1024Mi + limits: + cpu: 350m + memory: 1024Mi + + # each CL node have a dedicated PostgreSQL 11.15 + # use StatefulSet by setting: + # + # stateful: true + # capacity 10Gi + # + # if you are running long tests + db: + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + stateful: false + resources: + requests: + cpu: 1 + memory: 1024Mi + limits: + cpu: 1 + memory: 1024Mi + # default cluster shipped with latest Geth ( dev mode by default ) + geth: + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + version: v1.12.0 + wsrpc-port: 8546 + httprpc-port: 8544 + networkid: 1337 + blocktime: 1 + resources: + requests: + cpu: 1 + memory: 1024Mi + limits: + cpu: 1 + memory: 1024Mi + # mockserver is https://www.mock-server.com/where/kubernetes.html + # used to stub External Adapters + mockserver: + # image: "mockserver/mockserver" + # version: "mockserver-5.15.0" + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + enabled: true + releasenameOverride: mockserver + app: + runAsUser: 999 + readOnlyRootFilesystem: false + port: 1080 + resources: + requests: + cpu: 1 + memory: 1024Mi + limits: + cpu: 1 + memory: 1024Mi + runner: + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + stateful: false + resources: + requests: + cpu: 1 + memory: 512Mi + limits: + cpu: 1 + memory: 512Mi + affinity: { } + tolerations: [ ] + nodeSelector: { } + ingress: + enabled: false + className: "" + hosts: [ ] + tls: [ ] + annotations: { } + service: + type: NodePort + port: 8080 + + + # monitoring.coreos.com/v1 PodMonitor for each node + prometheusMonitor: true + + # deployment placement, standard helm stuff + podAnnotations: + nodeSelector: + tolerations: + affinity: profiles: # this replaces only "runner" pod, usable when you'd like to run some system level tests inside k8s diff --git a/charts/chainlink-cluster/templates/chainlink-deployment.yaml b/charts/chainlink-cluster/templates/chainlink-db-deployment.yaml similarity index 60% rename from charts/chainlink-cluster/templates/chainlink-deployment.yaml rename to charts/chainlink-cluster/templates/chainlink-db-deployment.yaml index b434c9894b0..f335130ea9f 100644 --- a/charts/chainlink-cluster/templates/chainlink-deployment.yaml +++ b/charts/chainlink-cluster/templates/chainlink-db-deployment.yaml @@ -6,10 +6,10 @@ kind: StatefulSet kind: Deployment {{ end }} metadata: - name: {{ $.Release.Name }}-{{ $cfg.name }} + name: {{ $.Release.Name }}-{{ $cfg.name }}-db spec: {{ if $.Values.db.stateful }} - serviceName: {{ $.Release.Name }}-{{ $cfg.name }}-service + serviceName: {{ $.Release.Name }}-db-${{ $cfg.name }} podManagementPolicy: Parallel volumeClaimTemplates: - metadata: @@ -23,14 +23,14 @@ spec: {{ end }} selector: matchLabels: - app: {{ $.Release.Name }} - instance: {{ $cfg.name }} + app: {{ $.Release.Name }}-db + instance: {{ $cfg.name }}-db release: {{ $.Release.Name }} template: metadata: labels: - app: {{ $.Release.Name }} - instance: {{ $cfg.name }} + app: {{ $.Release.Name }}-db + instance: {{ $cfg.name }}-db release: {{ $.Release.Name }} {{- range $key, $value := $.Values.labels }} {{ $key }}: {{ $value | quote }} @@ -44,11 +44,16 @@ spec: {{- end }} spec: volumes: + # TODO: breakout this config map into a separate one for the db. - name: {{ $.Release.Name }}-{{ $cfg.name }}-cm configMap: name: {{ $.Release.Name }}-{{ $cfg.name }}-cm + securityContext: + {{- toYaml $.Values.db.podSecurityContext | nindent 8 }} containers: - name: chainlink-db + securityContext: + {{- toYaml $.Values.db.securityContext | nindent 12 }} image: {{ default "postgres:11.15" $.Values.db.image }} command: - docker-entrypoint.sh @@ -114,60 +119,11 @@ spec: - mountPath: /docker-entrypoint-initdb.d/init.sql name: {{ $.Release.Name }}-{{ $cfg.name }}-cm subPath: init.sql - {{ if $.Values.db.stateful }} - volumeMounts: + {{ if $.Values.db.stateful }} - mountPath: /var/lib/postgresql/data name: postgres subPath: postgres-db - {{ end }} - - name: node - image: {{ default "public.ecr.aws/chainlink/chainlink" $cfg.image }} - imagePullPolicy: Always - command: ["bash", "-c", "while ! pg_isready --host 0.0.0.0 --port 5432; do echo \"waiting for database to start\"; sleep 1; done && chainlink -c /etc/node-secrets-volume/default.toml -c /etc/node-secrets-volume/overrides.toml -secrets /etc/node-secrets-volume/secrets.toml node start -d -p /etc/node-secrets-volume/node-password -a /etc/node-secrets-volume/apicredentials --vrfpassword=/etc/node-secrets-volume/apicredentials"] - ports: - - name: access - containerPort: {{ $.Values.chainlink.web_port }} - - name: p2p - containerPort: {{ $.Values.chainlink.p2p_port }} - env: - - name: CL_DATABASE_URL - value: postgresql://postgres:verylongdatabasepassword@0.0.0.0/chainlink?sslmode=disable - - name: CL_DEV - value: "false" - volumeMounts: - - name: {{ $.Release.Name }}-{{ $cfg.name }}-cm - mountPath: /etc/node-secrets-volume/ - livenessProbe: - httpGet: - path: /health - port: {{ $.Values.chainlink.web_port }} - initialDelaySeconds: 1 - periodSeconds: 5 - timeoutSeconds: 10 - readinessProbe: - httpGet: - path: /health - port: {{ $.Values.chainlink.web_port }} - initialDelaySeconds: 1 - periodSeconds: 5 - timeoutSeconds: 10 - startupProbe: - httpGet: - path: / - port: {{ $.Values.chainlink.web_port }} - initialDelaySeconds: 15 - periodSeconds: 5 - failureThreshold: 20 - {{ if (hasKey $.Values.chainlink "resources") }} - resources: - requests: - memory: {{ default "1024Mi" $.Values.chainlink.resources.requests.memory }} - cpu: {{ default "500m" $.Values.chainlink.resources.requests.cpu }} - limits: - memory: {{ default "1024Mi" $.Values.chainlink.resources.limits.memory }} - cpu: {{ default "500m" $.Values.chainlink.resources.limits.cpu }} - {{ else }} - {{ end }} + {{ end }} {{- with $.Values.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} @@ -181,4 +137,4 @@ spec: {{ toYaml . | indent 8 }} {{- end }} --- -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/chainlink-cluster/templates/chainlink-db-service.yaml b/charts/chainlink-cluster/templates/chainlink-db-service.yaml new file mode 100644 index 00000000000..f27bd9eab20 --- /dev/null +++ b/charts/chainlink-cluster/templates/chainlink-db-service.yaml @@ -0,0 +1,16 @@ +{{- range $cfg := .Values.chainlink.nodes }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $.Release.Name }}-db-{{ $cfg.name }} +spec: + selector: + app: {{ $.Release.Name }}-db + instance: {{ $cfg.name }}-db + release: {{ $.Release.Name }} + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 +--- +{{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/chainlink-node-deployment.yaml b/charts/chainlink-cluster/templates/chainlink-node-deployment.yaml new file mode 100644 index 00000000000..463453aff93 --- /dev/null +++ b/charts/chainlink-cluster/templates/chainlink-node-deployment.yaml @@ -0,0 +1,97 @@ +{{- range $cfg := .Values.chainlink.nodes }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $.Release.Name }}-{{ $cfg.name }} +spec: + selector: + matchLabels: + app: {{ $.Release.Name }} + instance: {{ $cfg.name }} + release: {{ $.Release.Name }} + template: + metadata: + labels: + app: {{ $.Release.Name }} + instance: {{ $cfg.name }} + release: {{ $.Release.Name }} + {{- range $key, $value := $.Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + annotations: + prometheus.io/scrape: 'true' + {{- range $key, $value := $.Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + spec: + volumes: + - name: {{ $.Release.Name }}-{{ $cfg.name }}-cm + configMap: + name: {{ $.Release.Name }}-{{ $cfg.name }}-cm + securityContext: + {{- toYaml $.Values.chainlink.podSecurityContext | nindent 8 }} + containers: + - name: node + securityContext: + {{- toYaml $.Values.chainlink.securityContext | nindent 12 }} + image: {{ default "public.ecr.aws/chainlink/chainlink" $cfg.image }} + imagePullPolicy: Always + command: ["bash", "-c", "while ! pg_isready -U postgres --host {{ $.Release.Name }}-db-{{ $cfg.name }} --port 5432; do echo \"waiting for database to start\"; sleep 1; done && chainlink -c /etc/node-secrets-volume/default.toml -c /etc/node-secrets-volume/overrides.toml -secrets /etc/node-secrets-volume/secrets.toml node start -d -p /etc/node-secrets-volume/node-password -a /etc/node-secrets-volume/apicredentials --vrfpassword=/etc/node-secrets-volume/apicredentials"] + ports: + - name: access + containerPort: {{ $.Values.chainlink.web_port }} + - name: p2p + containerPort: {{ $.Values.chainlink.p2p_port }} + env: + - name: CL_DATABASE_URL + value: postgresql://postgres:verylongdatabasepassword@{{ $.Release.Name }}-db-{{ $cfg.name }}/chainlink?sslmode=disable + - name: CL_DEV + value: "false" + volumeMounts: + - name: {{ $.Release.Name }}-{{ $cfg.name }}-cm + mountPath: /etc/node-secrets-volume/ + livenessProbe: + httpGet: + path: /health + port: {{ $.Values.chainlink.web_port }} + initialDelaySeconds: 1 + periodSeconds: 5 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: {{ $.Values.chainlink.web_port }} + initialDelaySeconds: 1 + periodSeconds: 5 + timeoutSeconds: 10 + startupProbe: + httpGet: + path: / + port: {{ $.Values.chainlink.web_port }} + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 20 + {{ if (hasKey $.Values.chainlink "resources") }} + resources: + requests: + memory: {{ default "1024Mi" $.Values.chainlink.resources.requests.memory }} + cpu: {{ default "500m" $.Values.chainlink.resources.requests.cpu }} + limits: + memory: {{ default "1024Mi" $.Values.chainlink.resources.limits.memory }} + cpu: {{ default "500m" $.Values.chainlink.resources.limits.cpu }} + {{ else }} + {{ end }} +{{- with $.Values.nodeSelector }} + nodeSelector: + {{ toYaml . | indent 8 }} +{{- end }} +{{- with $.Values.affinity }} + affinity: + {{ toYaml . | indent 8 }} +{{- end }} +{{- with $.Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +--- +{{- end }} diff --git a/charts/chainlink-cluster/templates/chainlink-service.yaml b/charts/chainlink-cluster/templates/chainlink-node-service.yaml similarity index 100% rename from charts/chainlink-cluster/templates/chainlink-service.yaml rename to charts/chainlink-cluster/templates/chainlink-node-service.yaml diff --git a/charts/chainlink-cluster/templates/geth-config-map.yaml b/charts/chainlink-cluster/templates/geth-config-map.yaml index f3369ba580f..022d9f2ea61 100644 --- a/charts/chainlink-cluster/templates/geth-config-map.yaml +++ b/charts/chainlink-cluster/templates/geth-config-map.yaml @@ -50,9 +50,9 @@ data: password.txt: | init.sh: | #!/bin/bash - if [ ! -d /root/.ethereum/keystore ]; then - echo "/root/.ethereum/keystore not found, running 'geth init'..." - geth init /root/ethconfig/genesis.json + if [ ! -d /app/.ethereum/keystore ]; then + echo "/app/.ethereum/keystore not found, running 'geth init'..." + geth init /app/ethconfig/genesis.json echo "...done!" fi diff --git a/charts/chainlink-cluster/templates/geth-deployment.yaml b/charts/chainlink-cluster/templates/geth-deployment.yaml index 11fb0cbee22..6948c4df288 100644 --- a/charts/chainlink-cluster/templates/geth-deployment.yaml +++ b/charts/chainlink-cluster/templates/geth-deployment.yaml @@ -22,31 +22,39 @@ spec: - name: configmap-volume configMap: name: geth-cm + - name: devchain-volume + emptyDir: {} + securityContext: + {{- toYaml $.Values.geth.podSecurityContext | nindent 8 }} containers: - name: geth-network + securityContext: + {{- toYaml $.Values.geth.securityContext | nindent 12 }} image: "{{ default "ethereum/client-go" .Values.geth.image }}:{{ default "stable" .Values.geth.version }}" - command: [ "sh", "./root/init.sh" ] + command: [ "sh", "/app/init.sh" ] volumeMounts: + - name: devchain-volume + mountPath: /app/.ethereum/devchain - name : configmap-volume - mountPath: /root/init.sh + mountPath: /app/init.sh subPath: init.sh - name: configmap-volume - mountPath: /root/config + mountPath: /app/config - name: configmap-volume - mountPath: /root/.ethereum/devchain/keystore/key1 + mountPath: /app/.ethereum/devchain/keystore/key1 subPath: key1 - name: configmap-volume - mountPath: /root/.ethereum/devchain/keystore/key2 + mountPath: /app/.ethereum/devchain/keystore/key2 subPath: key2 - name: configmap-volume - mountPath: /root/.ethereum/devchain/keystore/key3 + mountPath: /app/.ethereum/devchain/keystore/key3 subPath: key3 args: - '--dev' - '--password' - - '/root/config/password.txt' + - '/app/config/password.txt' - '--datadir' - - '/root/.ethereum/devchain' + - '/app/.ethereum/devchain' - '--unlock' - '0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266' - '--unlock' diff --git a/charts/chainlink-cluster/templates/mockserver-service.yaml b/charts/chainlink-cluster/templates/mockserver-service.yaml deleted file mode 100644 index f8ab78a84b5..00000000000 --- a/charts/chainlink-cluster/templates/mockserver-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{ if (hasKey .Values "mockserver") }} -apiVersion: v1 -kind: Service -metadata: - name: {{ .Release.Name }}-mockserver -spec: - selector: - app: {{ .Release.Name }}-mockserver - ports: - - name: serviceport - port: {{ default "1080" $.Values.mockserver.port}} - targetPort: serviceport - type: ClusterIP -{{ end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/mockserver.yaml b/charts/chainlink-cluster/templates/mockserver.yaml deleted file mode 100755 index 14c05d0acd5..00000000000 --- a/charts/chainlink-cluster/templates/mockserver.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{ if (hasKey .Values "mockserver") }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Release.Name }}-mockserver - labels: - app: {{ .Release.Name }}-mockserver -spec: - selector: - matchLabels: - app: {{ .Release.Name }}-mockserver - template: - metadata: -{{- if .Values.podAnnotations }} - annotations: -{{ toYaml .Values.podAnnotations | indent 8 }} -{{- end }} - name: {{ .Release.Name }}-mockserver - labels: - app: {{ .Release.Name }}-mockserver - spec: - containers: - - name: {{ .Release.Name }}-mockserver - image: {{ default "mockserver/mockserver" .Values.mockserver.image }}:{{ default "mockserver-5.15.0" .Values.mockserver.version }} - imagePullPolicy: IfNotPresent - securityContext: - runAsUser: 65534 # nonroot - readOnlyRootFilesystem: false - ports: - - name: serviceport - containerPort: {{ .Values.mockserver.port }} - protocol: TCP - env: - - name: LOG_LEVEL - value: "DEBUG" - - name: SERVER_PORT - value: {{ .Values.mockserver.port | quote }} - {{ if (hasKey $.Values.chainlink "resources") }} - resources: - requests: - memory: {{ default "1024Mi" $.Values.chainlink.resources.requests.memory }} - cpu: {{ default "500m" $.Values.chainlink.resources.requests.cpu }} - limits: - memory: {{ default "1024Mi" $.Values.chainlink.resources.limits.memory }} - cpu: {{ default "500m" $.Values.chainlink.resources.limits.cpu }} - {{ else }} - {{ end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{ toYaml . | indent 8 }} - {{- end }} -{{- end }} ---- \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/runner-deployment.yaml b/charts/chainlink-cluster/templates/runner-deployment.yaml index 5d9025b41c5..9d80ac1bfab 100644 --- a/charts/chainlink-cluster/templates/runner-deployment.yaml +++ b/charts/chainlink-cluster/templates/runner-deployment.yaml @@ -22,8 +22,12 @@ spec: annotations: prometheus.io/scrape: 'true' spec: + securityContext: + {{- toYaml $.Values.runner.podSecurityContext | nindent 8 }} containers: - name: runner + securityContext: + {{- toYaml $.Values.runner.securityContext | nindent 12 }} image: {{ default "public.ecr.aws/chainlink/chainlink" .Values.runner.image }} imagePullPolicy: Always command: [ "/bin/bash", "-c", "--" ] diff --git a/charts/chainlink-cluster/values-raw-helm.yaml b/charts/chainlink-cluster/values-raw-helm.yaml deleted file mode 100644 index 726a5347119..00000000000 --- a/charts/chainlink-cluster/values-raw-helm.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# override resources for keys "chainlink", "db", or "geth" if needed -# resources: -# requests: -# cpu: 350m -# memory: 1024Mi -# limits: -# cpu: 350m -# memory: 1024Mi -# images can be overriden for the same keys: -# image: ethereum/client-go -# version: stable -chainlink: - web_port: 6688 - p2p_port: 6690 - nodes: - - name: node-1 - image: "public.ecr.aws/chainlink/chainlink:latest" - # override default config per node - # for example, use OCRv2 P2P setup, the whole config -# toml: | -# RootDir = './clroot' -# [Log] -# JSONConsole = true -# Level = 'debug' -# [WebServer] -# AllowOrigins = '*' -# SecureCookies = false -# SessionTimeout = '999h0m0s' -# [OCR2] -# Enabled = true -# [P2P] -# [P2P.V2] -# Enabled = false -# AnnounceAddresses = [] -# DefaultBootstrappers = [] -# DeltaDial = '15s' -# DeltaReconcile = '1m0s' -# ListenAddresses = [] -# [[EVM]] -# ChainID = '1337' -# MinContractPayment = '0' -# [[EVM.Nodes]] -# Name = 'node-0' -# WSURL = 'ws://geth:8546' -# HTTPURL = 'http://geth:8544' -# [WebServer.TLS] -# HTTPSPort = 0 -# or use overridesToml to override some part of configuration -# overridesToml: | - - name: node-2 - - name: node-3 - - name: node-4 - - name: node-5 - - name: node-6 - resources: - requests: - cpu: 350m - memory: 1024Mi - limits: - cpu: 350m - memory: 1024Mi - -# each CL node have a dedicated PostgreSQL 11.15 -# use StatefulSet by setting: -# -# stateful: true -# capacity 10Gi -# -# if you are running long tests -db: - stateful: false - resources: - requests: - cpu: 1 - memory: 1024Mi - limits: - cpu: 1 - memory: 1024Mi -# default cluster shipped with latest Geth ( dev mode by default ) -geth: - version: v1.12.0 - wsrpc-port: 8546 - httprpc-port: 8544 - networkid: 1337 - blocktime: 1 - resources: - requests: - cpu: 1 - memory: 1024Mi - limits: - cpu: 1 - memory: 1024Mi -# mockserver is https://www.mock-server.com/where/kubernetes.html -# used to stub External Adapters -mockserver: - port: 1080 - resources: - requests: - cpu: 1 - memory: 1024Mi - limits: - cpu: 1 - memory: 1024Mi -runner: - stateful: false - resources: - requests: - cpu: 1 - memory: 512Mi - limits: - cpu: 1 - memory: 512Mi - affinity: { } - tolerations: [ ] - nodeSelector: { } - ingress: - enabled: false - className: "" - hosts: [ ] - tls: [ ] - annotations: { } - service: - type: NodePort - port: 8080 - - -# monitoring.coreos.com/v1 PodMonitor for each node -prometheusMonitor: false - -# deployment placement, standard helm stuff -podAnnotations: -nodeSelector: -tolerations: -affinity: diff --git a/charts/chainlink-cluster/values.yaml b/charts/chainlink-cluster/values.yaml new file mode 100644 index 00000000000..eb93e6cefcf --- /dev/null +++ b/charts/chainlink-cluster/values.yaml @@ -0,0 +1,178 @@ +# override resources for keys "chainlink", "db", or "geth" if needed +# resources: +# requests: +# cpu: 350m +# memory: 1024Mi +# limits: +# cpu: 350m +# memory: 1024Mi +# images can be overriden for the same keys: +# image: ethereum/client-go +# version: stable +chainlink: + podSecurityContext: + fsGroup: 14933 + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 14933 + runAsGroup: 14933 + web_port: 6688 + p2p_port: 6690 + nodes: + - name: node-1 + image: "public.ecr.aws/chainlink/chainlink:latest" + # override default config per node + # for example, use OCRv2 P2P setup, the whole config + # toml: | + # RootDir = './clroot' + # [Log] + # JSONConsole = true + # Level = 'debug' + # [WebServer] + # AllowOrigins = '*' + # SecureCookies = false + # SessionTimeout = '999h0m0s' + # [OCR2] + # Enabled = true + # [P2P] + # [P2P.V2] + # Enabled = false + # AnnounceAddresses = [] + # DefaultBootstrappers = [] + # DeltaDial = '15s' + # DeltaReconcile = '1m0s' + # ListenAddresses = [] + # [[EVM]] + # ChainID = '1337' + # MinContractPayment = '0' + # [[EVM.Nodes]] + # Name = 'node-0' + # WSURL = 'ws://geth:8546' + # HTTPURL = 'http://geth:8544' + # [WebServer.TLS] + # HTTPSPort = 0 + # or use overridesToml to override some part of configuration + # overridesToml: | + - name: node-2 + - name: node-3 + - name: node-4 + - name: node-5 + - name: node-6 + resources: + requests: + cpu: 350m + memory: 1024Mi + limits: + cpu: 350m + memory: 1024Mi + +# each CL node have a dedicated PostgreSQL 11.15 +# use StatefulSet by setting: +# +# stateful: true +# capacity 10Gi +# +# if you are running long tests +db: + podSecurityContext: + fsGroup: 999 + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + stateful: false + resources: + requests: + cpu: 1 + memory: 1024Mi + limits: + cpu: 1 + memory: 1024Mi +# default cluster shipped with latest Geth ( dev mode by default ) +geth: + podSecurityContext: + fsGroup: 999 + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + version: v1.12.0 + wsrpc-port: 8546 + httprpc-port: 8544 + networkid: 1337 + blocktime: 1 + resources: + requests: + cpu: 1 + memory: 1024Mi + limits: + cpu: 1 + memory: 1024Mi +# mockserver is https://www.mock-server.com/where/kubernetes.html +# used to stub External Adapters +mockserver: + enabled: true + releasenameOverride: mockserver + app: + runAsUser: 999 + readOnlyRootFilesystem: false + port: 1080 + resources: + requests: + cpu: 1 + memory: 1024Mi + limits: + cpu: 1 + memory: 1024Mi +runner: + podSecurityContext: + fsGroup: 999 + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + stateful: false + resources: + requests: + cpu: 1 + memory: 512Mi + limits: + cpu: 1 + memory: 512Mi + affinity: {} + tolerations: [] + nodeSelector: {} + ingress: + enabled: false + className: "" + hosts: [] + tls: [] + annotations: {} + service: + type: NodePort + port: 8080 + +# monitoring.coreos.com/v1 PodMonitor for each node +prometheusMonitor: true + +# deployment placement, standard helm stuff +podAnnotations: +nodeSelector: +tolerations: +affinity: diff --git a/core/chainlink.devspace.Dockerfile b/core/chainlink.devspace.Dockerfile index 9ec061ae40d..88d3cec16ad 100644 --- a/core/chainlink.devspace.Dockerfile +++ b/core/chainlink.devspace.Dockerfile @@ -20,7 +20,7 @@ RUN make install-chainlink # Final image: ubuntu with chainlink binary FROM golang:1.21-bullseye -ARG CHAINLINK_USER=root +ARG CHAINLINK_USER=chainlink ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y ca-certificates gnupg lsb-release curl