Fix Kafka CI (#863)

* Reenable kafka testing * Add kafka dev lib * Sync install python with devcontainer * Fix kafka local host setting * Drop set -u flag * Pin CI image dev sha * Add parallel flag to kafka * Fix proper exit status * Build librdkafka from source * Updated dev image sha * Remove coverage exclusions * Add new options to better emulate GHA * Reconfigure kafka networking Co-authored-by: Hannah Stepanek <hmstepanek@users.noreply.github.com> * Fix kafka ports on GHA * Run kafka tests serially * Separate kafka consumer groups * Put CI container makefile back * Remove confluent kafka Py27 for latest * Roll back ubuntu version update * Update dev ci sha --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hannah Stepanek <hmstepanek@users.noreply.github.com>
newrelic · Jul 14, 2023 · 2f580af · 2f580af
1 parent 53fc51a
commit 2f580af
Show file tree

Hide file tree

Showing 10 changed files with 129 additions and 118 deletions.
diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile
@@ -55,10 +55,22 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
 
+# Build librdkafka from source
+ARG LIBRDKAFKA_VERSION=2.1.1
+RUN cd /tmp && \
+    wget https://github.com/confluentinc/librdkafka/archive/refs/tags/v${LIBRDKAFKA_VERSION}.zip -O ./librdkafka.zip && \
+    unzip ./librdkafka.zip && \
+    rm ./librdkafka.zip && \
+    cd ./librdkafka-${LIBRDKAFKA_VERSION} && \
+    ./configure && \
+    make all install && \
+    cd /tmp && \
+    rm -rf ./librdkafka-${LIBRDKAFKA_VERSION}
+
 # Setup ODBC config
-RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini && \
-    sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini && \
-    sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini
+RUN sed -i 's|Driver=psqlodbca.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so|g' /etc/odbcinst.ini && \
+    sed -i 's|Driver=psqlodbcw.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbcw.so|g' /etc/odbcinst.ini && \
+    sed -i 's|Setup=libodbcpsqlS.so|Setup=/usr/lib/x86_64-linux-gnu/odbc/libodbcpsqlS.so|g' /etc/odbcinst.ini
 
 # Set the locale
 RUN locale-gen --no-purge en_US.UTF-8

diff --git a/.github/containers/Makefile b/.github/containers/Makefile
@@ -22,7 +22,9 @@ default: test
 .PHONY: build
 build:
 	@# Perform a shortened build for testing
-	@docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' $(MAKEFILE_DIR) -t ghcr.io/newrelic/newrelic-python-agent-ci:local
+	@docker build $(MAKEFILE_DIR) \
+		-t ghcr.io/newrelic/newrelic-python-agent-ci:local \
+		--build-arg='PYTHON_VERSIONS=3.10 2.7'
 
 .PHONY: test
 test: build
@@ -38,7 +40,9 @@ run: build
 	@docker run --rm -it \
 		--mount type=bind,source="$(REPO_ROOT)",target=/home/github/python-agent \
 		--workdir=/home/github/python-agent \
+		--add-host=host.docker.internal:host-gateway \
 		-e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \
 		-e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \
 		-e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \
+		-e GITHUB_ACTIONS="true" \
 		ghcr.io/newrelic/newrelic-python-agent-ci:local /bin/bash
diff --git a/.github/containers/install-python.sh b/.github/containers/install-python.sh
@@ -13,10 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
-
-SCRIPT_DIR=$(dirname "$0")
-PIP_REQUIREMENTS=$(cat /requirements.txt)
+set -eo pipefail
 
 main() {
     # Coerce space separated string to array
@@ -50,7 +47,7 @@ main() {
     pyenv global ${PYENV_VERSIONS[@]}
 
     # Install dependencies for main python installation
-    pyenv exec pip install --upgrade $PIP_REQUIREMENTS
+    pyenv exec pip install --upgrade -r /requirements.txt
 }
 
 main
diff --git a/.github/scripts/retry.sh b/.github/scripts/retry.sh
@@ -39,4 +39,4 @@ for i in $(seq 1 $retries); do
 done
 
 # Exit with status code of wrapped command
-exit $?
+exit $result
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -38,7 +38,7 @@ jobs:
       - elasticsearchserver08
       - gearman
       - grpc
-      #- kafka
+      - kafka
       - memcached
       - mongodb
       - mssql
@@ -119,7 +119,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -164,7 +164,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -209,7 +209,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -269,7 +269,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -332,7 +332,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -395,7 +395,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -453,7 +453,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -513,7 +513,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -571,7 +571,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -619,85 +619,75 @@ jobs:
           path: ./**/.coverage.*
           retention-days: 1
 
-  # kafka:
-  #   env:
-  #     TOTAL_GROUPS: 4
-
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       group-number: [1, 2, 3, 4]
-
-  #   runs-on: ubuntu-20.04
-  #   container:
-  #     image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
-  #     options: >-
-  #       --add-host=host.docker.internal:host-gateway
-  #   timeout-minutes: 30
-
-  #   services:
-  #     zookeeper:
-  #       image: bitnami/zookeeper:3.7
-  #       env:
-  #         ALLOW_ANONYMOUS_LOGIN: yes
-
-  #       ports:
-  #         - 2181:2181
-
-  #     kafka:
-  #       image: bitnami/kafka:3.2
-  #       ports:
-  #         - 8080:8080
-  #         - 8081:8081
-  #       env:
-  #         ALLOW_PLAINTEXT_LISTENER: yes
-  #         KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-  #         KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
-  #         KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081
-  #         KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081,
-  #         KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT
-  #         KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2
-
-  #   steps:
-  #     - uses: actions/checkout@v3
-
-  #     - name: Fetch git tags
-  #       run: |
-  #         git config --global --add safe.directory "$GITHUB_WORKSPACE"
-  #         git fetch --tags origin
-
-  #     # Special case packages
-  #     - name: Install librdkafka-dev
-  #       run: |
-  #         # Use lsb-release to find the codename of Ubuntu to use to install the correct library name
-  #         sudo apt-get update
-  #         sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime
-  #         sudo apt-get install -y wget gnupg2 software-properties-common
-  #         sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add -
-  #         sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
-  #         sudo apt-get update
-  #         sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2)
-
-  #     - name: Get Environments
-  #       id: get-envs
-  #       run: |
-  #         echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
-  #       env:
-  #         GROUP_NUMBER: ${{ matrix.group-number }}
-
-  #     - name: Test
-  #       run: |
-  #         tox -vv -e ${{ steps.get-envs.outputs.envs }}
-  #       env:
-  #         TOX_PARALLEL_NO_SPINNER: 1
-  #         PY_COLORS: 0
-
-  #     - name: Upload Coverage Artifacts
-  #       uses: actions/upload-artifact@v3
-  #       with:
-  #         name: coverage-${{ github.job }}-${{ strategy.job-index }}
-  #         path: ./**/.coverage.*
-  #         retention-days: 1
+  kafka:
+    env:
+      TOTAL_GROUPS: 4
+
+    strategy:
+      fail-fast: false
+      matrix:
+        group-number: [1, 2, 3, 4]
+
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
+      options: >-
+        --add-host=host.docker.internal:host-gateway
+    timeout-minutes: 30
+
+    services:
+      zookeeper:
+        image: bitnami/zookeeper:3.7
+        env:
+          ALLOW_ANONYMOUS_LOGIN: yes
+
+        ports:
+          - 2181:2181
+
+      kafka:
+        image: bitnami/kafka:3.2
+        ports:
+          - 8080:8080
+          - 8082:8082
+          - 8083:8083
+        env:
+          KAFKA_ENABLE_KRAFT: no
+          ALLOW_PLAINTEXT_LISTENER: yes
+          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+          KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
+          KAFKA_CFG_LISTENERS: L1://:8082,L2://:8083,L3://:8080
+          KAFKA_CFG_ADVERTISED_LISTENERS: L1://host.docker.internal:8082,L2://host.docker.internal:8083,L3://kafka:8080
+          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT,L3:PLAINTEXT
+          KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L3
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Fetch git tags
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git fetch --tags origin
+
+      - name: Get Environments
+        id: get-envs
+        run: |
+          echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
+        env:
+          GROUP_NUMBER: ${{ matrix.group-number }}
+
+      - name: Test
+        run: |
+          tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto
+        env:
+          TOX_PARALLEL_NO_SPINNER: 1
+          PY_COLORS: 0
+
+      - name: Upload Coverage Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-${{ github.job }}-${{ strategy.job-index }}
+          path: ./**/.coverage.*
+          retention-days: 1
 
   mongodb:
     env:
@@ -710,7 +700,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -768,7 +758,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -828,7 +818,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30
@@ -889,7 +879,7 @@ jobs:
 
     runs-on: ubuntu-20.04
     container:
-      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
       options: >-
         --add-host=host.docker.internal:host-gateway
     timeout-minutes: 30

diff --git a/codecov.yml b/codecov.yml
@@ -19,8 +19,5 @@ ignore:
   - "newrelic/hooks/database_oursql.py"
   - "newrelic/hooks/database_psycopg2ct.py"
   - "newrelic/hooks/datastore_umemcache.py"
-  # Temporarily disable kafka
-  - "newrelic/hooks/messagebroker_kafkapython.py"
-  - "newrelic/hooks/messagebroker_confluentkafka.py"
   - "newrelic/admin/*"
   - "newrelic/console.py"