From 47624f5a6dfa98fcc560c403c077e31ddf4a9200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Bobowski?= <145468486+sfc-gh-mbobowski@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:50:00 +0200 Subject: [PATCH] SNOW-947731 Remove deprecated avro-python3 package (#889) --- .github/workflows/End2EndTestApache.yml | 6 +++--- .github/workflows/End2EndTestConfluent.yml | 6 +++--- .github/workflows/IntegrationTest.yml | 4 ++-- .github/workflows/StressTest.yml | 6 +++--- test/test_suit/test_confluent_protobuf_protobuf.py | 8 +++----- test/test_suites.py | 1 - 6 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/End2EndTestApache.yml b/.github/workflows/End2EndTestApache.yml index 8cb05ac06..3888baa1a 100644 --- a/.github/workflows/End2EndTestApache.yml +++ b/.github/workflows/End2EndTestApache.yml @@ -43,9 +43,9 @@ jobs: run: | pip3 install --upgrade setuptools sudo apt-get install librdkafka-dev - pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0" - pip3 install avro-python3 kafka-python - pip3 install --upgrade protobuf==3.20.0 + pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2" + pip3 install avro kafka-python + pip3 install --upgrade protobuf==3.20.3 pip3 install --upgrade snowflake-connector-python==2.7.4 curl https://github.com/raw/helm/helm/master/scripts/get-helm-3 | bash sudo apt-get -y install jq vim diff --git a/.github/workflows/End2EndTestConfluent.yml b/.github/workflows/End2EndTestConfluent.yml index 4926cb3aa..01e25a852 100644 --- a/.github/workflows/End2EndTestConfluent.yml +++ b/.github/workflows/End2EndTestConfluent.yml @@ -43,9 +43,9 @@ jobs: run: | pip3 install --upgrade setuptools sudo apt-get install librdkafka-dev - pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0" - pip3 install avro-python3 kafka-python - pip3 install --upgrade protobuf==3.20.0 + pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2" + pip3 install avro kafka-python + pip3 install --upgrade protobuf==3.20.3 pip3 install --upgrade snowflake-connector-python==2.7.4 curl https://github.com/raw/helm/helm/master/scripts/get-helm-3 | bash sudo apt-get -y install jq vim diff --git a/.github/workflows/IntegrationTest.yml b/.github/workflows/IntegrationTest.yml index c2fef45e4..693112009 100644 --- a/.github/workflows/IntegrationTest.yml +++ b/.github/workflows/IntegrationTest.yml @@ -39,8 +39,8 @@ jobs: - name: Install Dependency run: | pip3 install --upgrade setuptools - pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0" - pip3 install avro-python3 kafka-python + pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2" + pip3 install avro kafka-python pip3 install protobuf pip3 install --upgrade snowflake-connector-python==2.7.4 curl https://github.com/raw/helm/helm/master/scripts/get-helm-3 | bash diff --git a/.github/workflows/StressTest.yml b/.github/workflows/StressTest.yml index 1bd7a0fcc..81935f380 100644 --- a/.github/workflows/StressTest.yml +++ b/.github/workflows/StressTest.yml @@ -36,9 +36,9 @@ jobs: run: | pip3 install --upgrade setuptools sudo apt-get install librdkafka-dev - pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0" - pip3 install avro-python3 kafka-python - pip3 install --upgrade protobuf==3.20.0 + pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2" + pip3 install avro kafka-python + pip3 install --upgrade protobuf==3.20.3 pip3 install --upgrade snowflake-connector-python==2.7.4 curl https://github.com/raw/helm/helm/master/scripts/get-helm-3 | bash sudo apt-get -y install jq diff --git a/test/test_suit/test_confluent_protobuf_protobuf.py b/test/test_suit/test_confluent_protobuf_protobuf.py index a34e5560d..0ad6ff508 100644 --- a/test/test_suit/test_confluent_protobuf_protobuf.py +++ b/test/test_suit/test_confluent_protobuf_protobuf.py @@ -29,11 +29,9 @@ def __init__(self, driver, nameSalt): self.sensor.uint64_val = (1 << 64) - 1 self.schema_registry_client = SchemaRegistryClient({'url': driver.schemaRegistryAddress}) - #uncomment for local tests - #self.keyProtobufSerializer = ProtobufSerializer(sensor_pb2.SensorReading, self.schema_registry_client, {'use.deprecated.format': True}) - #self.valueProtobufSerializer = ProtobufSerializer(sensor_pb2.SensorReading, self.schema_registry_client, {'use.deprecated.format': True}) - self.keyProtobufSerializer = ProtobufSerializer(sensor_pb2.SensorReading, self.schema_registry_client) - self.valueProtobufSerializer = ProtobufSerializer(sensor_pb2.SensorReading, self.schema_registry_client) + self.keyProtobufSerializer = ProtobufSerializer(sensor_pb2.SensorReading, self.schema_registry_client, {'use.deprecated.format': True}) + self.valueProtobufSerializer = ProtobufSerializer(sensor_pb2.SensorReading, self.schema_registry_client, {'use.deprecated.format': True}) + producer_conf = { 'bootstrap.servers': driver.kafkaAddress, 'key.serializer': self.keyProtobufSerializer, diff --git a/test/test_suites.py b/test/test_suites.py index bc15ed6ac..eae8d0264 100644 --- a/test/test_suites.py +++ b/test/test_suites.py @@ -203,7 +203,6 @@ def create_end_to_end_test_suites(driver, nameSalt, schemaRegistryAddress, testS test_instance=TestSchemaEvolutionAvroSR(driver, nameSalt), clean=True, run_in_confluent=True, run_in_apache=False )), - # SNOW-947731: Re-enable after avro-python3 package is updated in merge gate ("TestSchemaEvolutionAvroSRLogicalTypes", EndToEndTestSuite( test_instance=TestSchemaEvolutionAvroSRLogicalTypes(driver, nameSalt), clean=True, run_in_confluent=False, run_in_apache=False