Pipeline as Code integration for GitLab

.github/workflows/test-integration.yaml

@@ -15,12 +15,26 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
+      - uses: imjasonh/setup-ko@v0.6
       - name: Install Binaries
         run: ./hack/binaries.sh
-      - name: Allocate Cluster
-        run: ./hack/allocate.sh
+      - name: Setup testing func image
+        run: ./hack/create-testing-func-image.sh
       - name: Local Registry
         run: ./hack/registry.sh
+      - name: Allocate Cluster
+        run: ./hack/allocate.sh
+      - name: Install Tekton
+        run: ./hack/tekton.sh
+      - name: Install Pipelines as Code
+        run: ./hack/install-pac.sh
+      - name: Install Gitlab
+        run: |
+          export GITLAB_ROOT_PASSWORD=nbusr123
+          echo "GITLAB_ROOT_PASSWORD=${GITLAB_ROOT_PASSWORD}" >> "$GITHUB_ENV"
+          ./hack/install-gitlab.sh
+      - name: Patch Hosts
+        run: ./hack/patch-hosts.sh
       - name: Integration Test
         run: make test-integration
       - uses: codecov/codecov-action@v3

go.mod

@@ -37,7 +37,9 @@ require (
 	github.com/tektoncd/cli v0.31.0
 	github.com/tektoncd/pipeline v0.47.0
 	github.com/whilp/git-urls v1.0.0
+	github.com/xanzy/go-gitlab v0.83.0
 	golang.org/x/crypto v0.8.0
+	golang.org/x/net v0.9.0
 	golang.org/x/oauth2 v0.7.0
 	golang.org/x/sync v0.1.0
 	golang.org/x/term v0.8.0
@@ -142,7 +144,9 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
 	github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
@@ -220,7 +224,6 @@ require (
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect
 	golang.org/x/mod v0.9.0 // indirect
-	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -955,10 +955,12 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
+github.com/hashicorp/go-hclog v1.3.1 h1:vDwF1DFNZhntP4DAjuTpOw3uEgMUpXh1pB5fW9DqHpo=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@@ -970,6 +972,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
+github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -1644,6 +1648,8 @@ github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT
 github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/xanzy/go-gitlab v0.83.0 h1:37p0MpTPNbsTMKX/JnmJtY8Ch1sFiJzVF342+RvZEGw=
+github.com/xanzy/go-gitlab v0.83.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=

hack/allocate.sh

@@ -64,6 +64,9 @@ nodes:
     - containerPort: 433
       hostPort: 443
       listenAddress: "127.0.0.1"
+    - containerPort: 30022
+      hostPort: 30022
+      listenAddress: "127.0.0.1"
 containerdConfigPatches:
 - |-
   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:50000"]

hack/install-gitlab.sh

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+
+function install_gitlab() {
+  local -r gitlab_host="gitlab.127.0.0.1.sslip.io"
+
+  kubectl apply -f - <<EOF
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: gitlab
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: gitlab
+  namespace: gitlab
+  labels:
+    app: gitlab
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: gitlab
+  namespace: gitlab
+  labels:
+    app.kubernetes.io/name: gitlab
+spec:
+  containers:
+    - name: gitlab
+      image: gitlab/gitlab-ce:latest
+      volumeMounts:
+        - name: gitlab
+          subPath: config
+          mountPath: /etc/gitlab
+        - name: gitlab
+          subPath: logs
+          mountPath: /var/log/gitlab
+        - name: gitlab
+          subPath: data
+          mountPath: /var/opt/gitlab
+      env:
+        - name: GITLAB_ROOT_PASSWORD
+          value: ${GITLAB_ROOT_PASSWORD}
+        - name: GITLAB_OMNIBUS_CONFIG
+          value: |
+            external_url 'http://${gitlab_host}'
+            gitlab_rails['gitlab_shell_ssh_port'] = 30022
+            gitlab_rails['gitlab_email_enabled'] = false
+            puma['worker_processes'] = 0
+            sidekiq['max_concurrency'] = 1
+            prometheus_monitoring['enable'] = false
+            gitlab_rails['env'] = {
+              'MALLOC_CONF' => 'dirty_decay_ms:1000,muzzy_decay_ms:1000'
+            }
+            gitaly['configuration'] = {
+              ruby_max_rss: 200_000_000,
+              concurrency: [
+                {
+                  rpc: "/gitaly.SmartHTTPService/PostReceivePack",
+                  max_per_repo: 1
+                }, {
+                  rpc: "/gitaly.SSHService/SSHUploadPack",
+                  max_per_repo: 1
+                }
+              ]
+            }
+            gitaly['env'] = {
+              'MALLOC_CONF' => 'dirty_decay_ms:1000,muzzy_decay_ms:1000',
+              'GITALY_COMMAND_SPAWN_MAX_PARALLEL' => '1'
+            }
+      ports:
+        - containerPort: 80
+          name: http
+        - containerPort: 22
+          name: ssh
+      resources:
+        requests:
+          memory: "1024Mi"
+        limits:
+          memory: "2048Mi"
+  volumes:
+    - name: gitlab
+      persistentVolumeClaim:
+        claimName: gitlab
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitlab-internal
+  namespace: gitlab
+spec:
+  selector:
+    app.kubernetes.io/name: gitlab
+  ports:
+    - name: http
+      protocol: TCP
+      port: 80
+      targetPort: http
+    - name: ssh
+      protocol: TCP
+      port: 30022
+      targetPort: ssh
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: gitlab-external-ssh
+  namespace: gitlab
+spec:
+  selector:
+    app.kubernetes.io/name: gitlab
+  ports:
+    - name: ssh
+      protocol: TCP
+      port: 30022
+      targetPort: ssh
+      nodePort: 30022
+  type: NodePort
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gitlab
+  namespace: gitlab
+spec:
+  ingressClassName: contour-external
+  rules:
+    - host: ${gitlab_host}
+      http:
+        paths:
+          - backend:
+              service:
+                name: gitlab-internal
+                port:
+                  number: 80
+            pathType: Prefix
+            path: /
+
+EOF
+
+  sleep 1
+  kubectl wait pod --for=condition=Ready -l '!job-name' -n gitlab --timeout=5m
+
+  echo '::group::Waiting for Gitlab'
+  if ! curl --retry 120 -f --retry-all-errors --retry-delay 5 "${gitlab_host}"; then
+    kubectl logs pod/gitlab -n gitlab
+    echo '::endgroup::'
+    return 1
+  fi
+  echo '::endgroup::'
+}
+
+if [ "$0" = "${BASH_SOURCE[0]}" ]; then
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  function main() {
+    install_gitlab
+  }
+  main "$@"
+fi

hack/install-pac.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+function install_pac() {
+    local -r pac_ctr_host="pac-ctr.127.0.0.1.sslip.io"
+    local -r pac_version="v0.17.1"
+
+    # Install Pipelines as Code
+    kubectl apply -f "https://github.com/raw/openshift-pipelines/pipelines-as-code/release-${pac_version}/release.k8s.yaml"
+    sleep 5
+    kubectl wait pod --for=condition=Ready -l '!job-name' -n pipelines-as-code --timeout=5m
+
+    # Install ingress for the PaC controller. This is used by VCS Webhooks.
+    kubectl apply -f - << EOF
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: pipelines-as-code
+  namespace: pipelines-as-code
+spec:
+  ingressClassName: contour-external
+  rules:
+  - host: ${pac_ctr_host}
+    http:
+      paths:
+      - backend:
+          service:
+            name: pipelines-as-code-controller
+            port:
+              number: 8080
+        pathType: Prefix
+        path: /
+EOF
+}
+
+if [ "$0" = "${BASH_SOURCE[0]}" ]; then
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  function main() {
+      install_pac
+  }
+  main "$@"
+fi

hack/patch-hosts.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# This script creates a DNS A records for '127.0.0.1.sslip.io' and '*.127.0.0.1.sslip.io' pointing to the cluster node.
+
+function patch_hosts() {
+  local cluster_node_addr
+
+  cluster_node_addr="$(docker container inspect func-control-plane | jq ".[0].NetworkSettings.Networks.kind.IPAddress" -r)"
+
+  kubectl patch cm/coredns -n kube-system --patch-file /dev/stdin <<EOF
+{
+  "data": {
+    "Corefile": ".:53 {\n    errors\n    health {\n       lameduck 5s\n    }\n    ready\n    kubernetes cluster.local in-addr.arpa ip6.arpa {\n       pods insecure\n       fallthrough in-addr.arpa ip6.arpa\n       ttl 30\n    }\n    file /etc/coredns/example.db 127.0.0.1.sslip.io\n    prometheus :9153\n    forward . /etc/resolv.conf {\n       max_concurrent 1000\n    }\n    cache 30\n    loop\n    reload\n    loadbalance\n}\n",
+    "example.db": "; 127.0.0.1.sslip.io test file\n127.0.0.1.sslip.io.            IN      SOA     sns.dns.icann.org. noc.dns.icann.org. 2015082541 7200 3600 1209600 3600\n127.0.0.1.sslip.io.            IN      A       ${cluster_node_addr}\n*.127.0.0.1.sslip.io.          IN      A       ${cluster_node_addr}\n"
+  }
+}
+EOF
+
+  kubectl patch deploy/coredns -n kube-system --patch-file /dev/stdin <<EOF
+{
+  "spec": {
+    "template": {
+      "spec": {
+        "\$setElementOrder/volumes": [
+          {
+            "name": "config-volume"
+          }
+        ],
+        "volumes": [
+          {
+            "\$retainKeys": [
+              "configMap",
+              "name"
+            ],
+            "configMap": {
+              "items": [
+                {
+                  "key": "Corefile",
+                  "path": "Corefile"
+                },
+                {
+                  "key": "example.db",
+                  "path": "example.db"
+                }
+              ]
+            },
+            "name": "config-volume"
+          }
+        ]
+      }
+    }
+  }
+}
+EOF
+  sleep 1
+  kubectl wait pod --for=condition=Ready -l '!job-name' -n kube-system --timeout=15s
+}
+
+if [ "$0" = "${BASH_SOURCE[0]}" ]; then
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  function main() {
+    patch_hosts
+  }
+  main "$@"
+fi