Prep for CRIB (#11418)

* Make CICD work with new ECRs (#495) * Fix ECR and publish on PRs (#496) * Remove test file * Add mockserver to helm chart dependency * Add default values file * Add security contexts * Fix ref to values * Use head SHA instead of commit SHA for image tagging * Use emptyDir mount for postgres to resolve perm issues * Fix volume mounts on postgres * Fix /tmp writes and mount /tmp emptyDir * Remove custom uid * Run as same uid as postgres * Test DOCKER_METADATA_PR_HEAD_SHA disable for root image * Upgrade docker metadata action * Do a full clone * Avoid emptyDir mounts and update gid * Use head SHA when on a PR as default input * Downgrade metata action to latest on prev major * Override sha tag with HEAD value * Remove old sha tag default * Attempt to fix sha tag * Fix syntax error * Remove DOCKER_METADATA_PR_HEAD_SHA env * Refactor shared var output * Revert "Refactor shared var output" This reverts commit 2362fe6473974d4723cf5294cccec7090ab07a0f. * Revert "Remove DOCKER_METADATA_PR_HEAD_SHA env" This reverts commit a7bd01e544ad97603147ccb33576e8006056e930. * Try to make things work again * Set the host to localhost for K8s * Add user to pg_isready * Make securityContext container specific * Create init container to create /clroot * Fix volumes * Add security context to init container * Skip the chown * Remove init container * Breakout DB into its own deployment and service * Fix typo * Update localhost to db service dns * Fix path to script * Change geth /root path to /app * Create emptyDir volume for geth devchain dir * Remove full clone * Add correct inputs to mockserver * merge with develop * cleanup & verify * Add fixes for /chainlink * Remove unused env var * Set back to hardcoded repo name * Reset values back to pre rebase changes * Remove mockserver templates in favor of subchart * fix connect.toml and rename mockserver connection * Backout triggering on PR * Create new build-publish workflow for chainlink-untrusted (from PR) * Reset workflow back * Make step name accurate --------- Co-authored-by: skudasov <f4hrenh9it@gmail.com>
smartcontractkit · Dec 1, 2023 · bb03a45 · bb03a45
1 parent e140618
commit bb03a45
Show file tree

Hide file tree

Showing 20 changed files with 554 additions and 349 deletions.
diff --git a/.github/actions/build-sign-publish-chainlink/action.yml b/.github/actions/build-sign-publish-chainlink/action.yml
@@ -34,7 +34,7 @@ inputs:
     required: false
   git-commit-sha:
     description: Git commit SHA used as metadata when building the application (appears in logs)
-    default: ${{ github.sha }}
+    default: ${{ github.event.pull_request.head.sha || github.sha }}
     required: false
   aws-role-to-assume:
     description: The AWS role to assume as the CD user, if any. Used in configuring the docker/login-action
@@ -73,7 +73,7 @@ runs:
   using: composite
   steps:
     - name: Set shared variables
-      shell: sh
+      shell: bash
       # See https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
       run: |
         SHARED_IMAGES=${{ inputs.ecr-hostname }}/${{ inputs.ecr-image-name }}
@@ -122,7 +122,9 @@ runs:
 
     - name: Generate docker metadata for root image
       id: meta-root
-      uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v4.4.4
+      uses: docker/metadata-action@2c0bd771b40637d97bf205cbccdd294a32112176 # v4.5.0
+      env:
+        DOCKER_METADATA_PR_HEAD_SHA: "true"
       with:
         # list of Docker images to use as base name for tags
         images: ${{ env.shared-images }}
@@ -164,7 +166,9 @@ runs:
 
     - name: Generate docker metadata for non-root image
       id: meta-nonroot
-      uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e # v4.4.4
+      uses: docker/metadata-action@2c0bd771b40637d97bf205cbccdd294a32112176 # v4.5.0
+      env:
+        DOCKER_METADATA_PR_HEAD_SHA: "true"
       with:
         flavor: |
           latest=auto

diff --git a/.github/workflows/automation-ondemand-tests.yml b/.github/workflows/automation-ondemand-tests.yml
@@ -115,7 +115,7 @@ jobs:
       pull-requests: write
       id-token: write
       contents: read
-    needs: [ build-chainlink, build-test-image ]
+    needs: [build-chainlink, build-test-image]
     env:
       CHAINLINK_COMMIT_SHA: ${{ github.sha }}
       CHAINLINK_ENV_USER: ${{ github.actor }}

diff --git a/.github/workflows/build-publish-pr.yml b/.github/workflows/build-publish-pr.yml
@@ -0,0 +1,44 @@
+name: "Build and Publish from PR"
+
+##
+# This workflow builds and publishes a Docker image for Chainlink from a PR.
+# It doesn't use an environment, has its own special IAM role, does not sign
+# the image, and publishes to a special ECR repo.
+##
+
+on:
+  pull_request:
+
+jobs:
+  build-publish-untrusted:
+    if: ${{ ! startsWith(github.ref_name, 'release/') }}
+    runs-on: ubuntu-20.04
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Build and publish chainlink image
+        uses: ./.github/actions/build-sign-publish-chainlink
+        with:
+          publish: true
+          aws-role-to-assume: ${{ secrets.AWS_OIDC_IAM_ROLE_PUBLISH_PR_ARN }}
+          aws-role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS_DEFAULT }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          sign-images: false
+          ecr-hostname: ${{ secrets.AWS_SDLC_ECR_HOSTNAME }}
+          ecr-image-name: chainlink-untrusted
+          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
+          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
+
+      - name: Collect Metrics
+        if: always()
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: build-publish-untrusted
+        continue-on-error: true
diff --git a/charts/chainlink-cluster/Chart.yaml b/charts/chainlink-cluster/Chart.yaml
@@ -2,4 +2,9 @@ apiVersion: v1
 name: chainlink-cluster
 description: Chainlink nodes cluster
 version: 0.1.3
-appVersion: '2.6.0'
+appVersion: "2.6.0"
+dependencies:
+  - name: mockserver
+    version: "5.14.0"
+    repository: "@mockserver"
+    condition: mockserver.enabled
diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md
@@ -74,10 +74,10 @@ After that all the changes will be synced automatically
 Check `.profiles` to understand what is uploaded in profiles `runner` and `node`
 
 # Helm
-If you would like to use `helm` directly, please uncomment data in `values-raw-helm.yaml`
+If you would like to use `helm` directly, please uncomment data in `values.yaml`
 ## Install from local files
 ```
-helm install -f values-raw-helm.yaml cl-cluster .
+helm install -f values.yaml cl-cluster .
 ```
 Forward all apps (in another terminal)
 ```
@@ -99,7 +99,7 @@ kubectl config set-context --current --namespace cl-cluster
 
 Install
 ```
-helm install -f values-raw-helm.yaml cl-cluster chainlink-cluster/chainlink-cluster --version v0.1.2
+helm install -f values.yaml cl-cluster .
 ```
 
 ## Create a new release

diff --git a/charts/chainlink-cluster/connect.toml b/charts/chainlink-cluster/connect.toml
@@ -9,4 +9,4 @@ cl_node_url_template = "http://app-node-%d:6688"
 cl_node_internal_dns_record_template = "app-node-%d"
 cl_node_user = "notreal@fakeemail.ch"
 cl_node_password = "fj293fbBnlQ!f9vNs"
-mockserver_url = "http://app-mockserver:1080"
+mockserver_url = "http://mockserver:1080"
diff --git a/charts/chainlink-cluster/dashboard/dashboard.go b/charts/chainlink-cluster/dashboard/dashboard.go
@@ -350,48 +350,6 @@ func (m *CLClusterDashboard) generate() error {
 				),
 			),
 		),
-		// logs
-		dashboard.Row(
-			"Logs",
-			row.Collapse(),
-			row.WithTimeSeries(
-				"Log Counters",
-				timeseries.Span(12),
-				timeseries.Height("200px"),
-				timeseries.DataSource(m.PrometheusDataSourceName),
-				timeseries.WithPrometheusTarget(
-					`log_panic_count{namespace="${namespace}"}`,
-					prometheus.Legend("{{pod}} - panic"),
-				),
-				timeseries.WithPrometheusTarget(
-					`log_fatal_count{namespace="${namespace}"}`,
-					prometheus.Legend("{{pod}} - fatal"),
-				),
-				timeseries.WithPrometheusTarget(
-					`log_critical_count{namespace="${namespace}"}`,
-					prometheus.Legend("{{pod}} - critical"),
-				),
-				timeseries.WithPrometheusTarget(
-					`log_warn_count{namespace="${namespace}"}`,
-					prometheus.Legend("{{pod}} - warn"),
-				),
-				timeseries.WithPrometheusTarget(
-					`log_error_count{namespace="${namespace}"}`,
-					prometheus.Legend("{{pod}} - error"),
-				),
-			),
-			m.logsRowOption("All errors", `
-			{namespace="${namespace}", app="app", container="node"} 
-			| json 
-			| level="error" 
-			|  line_format "{{ .instance }} {{ .level }} {{ .ts }} {{ .logger }} {{ .caller }} {{ .msg }} {{ .version }} {{ .nodeTier }} {{ .nodeName }} {{ .node }} {{ .evmChainID }} {{ .nodeOrder }} {{ .mode }} {{ .nodeState }} {{ .sentryEventID }} {{ .stacktrace }}"`),
-			m.logsRowOption("Node 1", `{namespace="${namespace}", app="app", instance="node-1", container="node"}`),
-			m.logsRowOption("Node 2", `{namespace="${namespace}", app="app", instance="node-2", container="node"}`),
-			m.logsRowOption("Node 3", `{namespace="${namespace}", app="app", instance="node-3", container="node"}`),
-			m.logsRowOption("Node 4", `{namespace="${namespace}", app="app", instance="node-4", container="node"}`),
-			m.logsRowOption("Node 5", `{namespace="${namespace}", app="app", instance="node-5", container="node"}`),
-			m.logsRowOption("Node 6", `{namespace="${namespace}", app="app", instance="node-6", container="node"}`),
-		),
 		// HeadTracker
 		dashboard.Row("Head tracker",
 			row.Collapse(),

diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml
@@ -40,26 +40,56 @@ deployments:
       # they can be defined the same way in values.yml
       # devspace merging this "values" and "values.yml" before deploy
       values:
-        runner:
-          image: ${DEVSPACE_IMAGE}
-          stateful: false
-        geth:
-          version: v1.12.0
-          wsrpc-port: 8546
-          httprpc-port: 8544
-          networkid: 1337
-          blocktime: 1
-        mockserver:
-          port: 1080
-        db:
-          stateful: false
+        podSecurityContext:
+          fsGroup: 999
+
         chainlink:
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 14933
+            runAsGroup: 999
           web_port: 6688
           p2p_port: 6690
           nodes:
             - name: node-1
               image: ${DEVSPACE_IMAGE}
               version: latest
+              # override default config per node
+              # for example, use OCRv2 P2P setup, the whole config
+              #      toml: |
+              #        RootDir = './clroot'
+              #        [Log]
+              #        JSONConsole = true
+              #        Level = 'debug'
+              #        [WebServer]
+              #        AllowOrigins = '*'
+              #        SecureCookies = false
+              #        SessionTimeout = '999h0m0s'
+              #        [OCR2]
+              #        Enabled = true
+              #        [P2P]
+              #        [P2P.V2]
+              #        Enabled = false
+              #        AnnounceAddresses = []
+              #        DefaultBootstrappers = []
+              #        DeltaDial = '15s'
+              #        DeltaReconcile = '1m0s'
+              #        ListenAddresses = []
+              #        [[EVM]]
+              #        ChainID = '1337'
+              #        MinContractPayment = '0'
+              #        [[EVM.Nodes]]
+              #        Name = 'node-0'
+              #        WSURL = 'ws://geth:8546'
+              #        HTTPURL = 'http://geth:8544'
+              #        [WebServer.TLS]
+              #        HTTPSPort = 0
+              # or use overridesToml to override some part of configuration
+              #      overridesToml: |
             - name: node-2
               image: ${DEVSPACE_IMAGE}
               version: latest
@@ -75,11 +105,125 @@ deployments:
             - name: node-6
               image: ${DEVSPACE_IMAGE}
               version: latest
-        prometheusMonitor: "true"
-        podAnnotations: { }
-        nodeSelector: { }
-        tolerations: [ ]
-        affinity: { }
+          resources:
+            requests:
+              cpu: 350m
+              memory: 1024Mi
+            limits:
+              cpu: 350m
+              memory: 1024Mi
+
+        # each CL node have a dedicated PostgreSQL 11.15
+        # use StatefulSet by setting:
+        #
+        # stateful: true
+        # capacity 10Gi
+        #
+        # if you are running long tests
+        db:
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            runAsGroup: 999
+          stateful: false
+          resources:
+            requests:
+              cpu: 1
+              memory: 1024Mi
+            limits:
+              cpu: 1
+              memory: 1024Mi
+        # default cluster shipped with latest Geth ( dev mode by default )
+        geth:
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            runAsGroup: 999
+          version: v1.12.0
+          wsrpc-port: 8546
+          httprpc-port: 8544
+          networkid: 1337
+          blocktime: 1
+          resources:
+            requests:
+              cpu: 1
+              memory: 1024Mi
+            limits:
+              cpu: 1
+              memory: 1024Mi
+        # mockserver is https://www.mock-server.com/where/kubernetes.html
+        # used to stub External Adapters
+        mockserver:
+          #  image: "mockserver/mockserver"
+          #  version: "mockserver-5.15.0"
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            runAsGroup: 999
+          enabled: true
+          releasenameOverride: mockserver
+          app:
+            runAsUser: 999
+            readOnlyRootFilesystem: false
+          port: 1080
+          resources:
+            requests:
+              cpu: 1
+              memory: 1024Mi
+            limits:
+              cpu: 1
+              memory: 1024Mi
+        runner:
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            runAsGroup: 999
+          stateful: false
+          resources:
+            requests:
+              cpu: 1
+              memory: 512Mi
+            limits:
+              cpu: 1
+              memory: 512Mi
+          affinity: { }
+          tolerations: [ ]
+          nodeSelector: { }
+          ingress:
+            enabled: false
+            className: ""
+            hosts: [ ]
+            tls: [ ]
+            annotations: { }
+          service:
+            type: NodePort
+            port: 8080
+
+
+        # monitoring.coreos.com/v1 PodMonitor for each node
+        prometheusMonitor: true
+
+        # deployment placement, standard helm stuff
+        podAnnotations:
+        nodeSelector:
+        tolerations:
+        affinity:
 
 profiles:
   # this replaces only "runner" pod, usable when you'd like to run some system level tests inside k8s