NIFI-12178 add integration-tests and docker-tests github actions #4

Workflow file for this run

.github/workflows/integration-tests.yml at da00ccf

	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	name: integration-tests

	on:
	# Run every day at 03:00
	schedule:
	- cron: "0 3 * * *"
	push:
	paths:
	- '.github/workflows/integration-tests.yml'
	- '/test//*IT.java'
	- '/test//*ITCase.java'
	- '/test//IT*.java'
	pull_request:
	paths:
	- '.github/workflows/integration-tests.yml'
	- '/test//*IT.java'
	- '/test//*ITCase.java'
	- '/test//IT*.java'

	env:
	DEFAULT_MAVEN_OPTS: >-
	-Xmx4g
	-XX:ReservedCodeCacheSize=1g
	-XX:+UseG1GC
	-Dorg.slf4j.simpleLogger.defaultLogLevel=WARN
	-Daether.connector.http.retryHandler.count=5
	-Daether.connector.http.connectionMaxTtl=30
	-Duser.language=en
	-Duser.country=GB
	-Duser.timezone=UTC
	MAVEN_COMMAND: >-
	./mvnw
	-V
	-nsu
	-ntp
	-ff

	# skip assemblies and modules covered separately by the system-tests workflow
	# skip unit tests (surefire), run integration tests (failsafe), python ITs included as part of the system-tests workflow
	MAVEN_BUILD_ARGUMENTS: >-
	-T 2.0C
	-D failsafe.failIfNoSpecifiedTests=false
	package
	-P skip-unit-tests
	-P integration-tests
	-P nifi-registry-integration-tests
	package
	MAVEN_BUILD_EXCLUDE_PROJECTS: >-
	-pl -:minifi-assembly
	-pl -:minifi-c2-assembly
	-pl -:nifi-assembly
	-pl -:nifi-toolkit-assembly
	-pl -:nifi-registry-assembly
	-pl -:nifi-registry-toolkit-assembly
	-pl -:nifi-runtime-manifest
	-pl -:nifi-runtime-manifest-test
	-pl -:nifi-stateless-assembly
	-pl -:nifi-stateless-processor-tests
	-pl -:nifi-stateless-system-test-suite
	-pl -:nifi-system-test-suite
	-pl -:nifi-nar-provider-assembly
	-pl -:nifi-py4j-integration-tests
	# skip individual failing ITs (some won't work in a CI environment)
	# always run the (failsafe default) IT, IT, *ITCase integration-tests
	MAVEN_FAILSAFE_SKIP_ITS: >-
	IT, IT, *ITCase,
	!AwsSecretsManagerSensitivePropertyProviderIT,
	!AwsKmsSensitivePropertyProviderIT,
	!AzureKeyVaultKeySensitivePropertyProviderIT,
	!GcpKmsSensitivePropertyProviderIT,
	!StandardHashiCorpVaultCommunicationServiceIT,
	!ConversionWithSchemaInferenceIT#testXmlToAvro,
	!ConversionWithSchemaInferenceIT#testXmlToJson,
	!ConversionWithSchemaInferenceIT#testJsonToCsv,
	!ConversionWithSchemaInferenceIT#testCsvToXml,
	!ConversionWithSchemaInferenceIT#testXmlToAvroToXml,
	!ConversionWithSchemaInferenceIT#testCsvToAvro,
	!ConversionWithSchemaInferenceIT#testCsvToJson,
	!ConversionWithSchemaInferenceIT#testJsonToAvroToJson,
	!ConversionWithSchemaInferenceIT#testCsvToAvroToCsv,
	!ConversionWithExplicitSchemaIT#testJsonToCsv,
	!ConversionWithExplicitSchemaIT#testJsonToAvro,
	!ConversionWithExplicitSchemaIT#testJsonToAvroToJson,
	!ITConsumeKinesisStreamEndpointOverride,
	!ITConsumeKinesisStreamConnectAWS,
	!ITPutKinesisStream,
	!ITPutKinesisStreamWithEndpointOverride#testIntegrationSuccess,
	!ITPutKinesisFirehoseWithEndpointOverride#testIntegrationSuccess,
	!ITPutKinesisFirehose,
	!ITTagS3Object,
	!ITFetchS3Object,
	!ITDeleteS3Object,
	!ITPutS3Object,
	!ITListS3,
	!ITPutSNS#testPublish,
	!ITPutSNS#testPublishWithCredentialsProviderService,
	!ITDeleteSQS,
	!ITGetSQS,
	!ITPutSQS,
	!ITPutLambda,
	!GetMongoIT#testDatabaseEL,
	!PutGridFSIT#testFileNameAndHashUniqueness,
	!PutGridFSIT#testHashUniqueness,
	!ITPutAzureCosmosDBRecord,
	!ITListAzureBlobStorage_v12,
	!ITPutAzureDataLakeStorage,
	!ITFetchAzureDataLakeStorage,
	!ITListAzureDataLakeStorage,
	!ITFetchAzureBlobStorage_v12,
	!ITPutAzureBlobStorage_v12,
	!ITDeleteAzureDataLakeStorage,
	!ITDeleteAzureBlobStorage_v12,
	!ITMoveAzureDataLakeStorage,
	!AzureGraphUserGroupProviderIT,
	!JMSPublisherConsumerIT#validateMessageRedeliveryWhenNotAcked,
	!ITPutInfluxDB,
	!ITExecuteInfluxDBQuery,
	!GremlinClientServiceYamlSettingsAndBytecodeIT,
	!GremlinClientServiceControllerSettingsIT,
	!ITestConsumeEmail#validateUrl,
	!PrometheusReportingTaskIT#testNullLabel,
	!SnowflakeConnectionPoolIT,
	!SalesforceRestServiceIT,
	!QuerySalesforceObjectIT,
	!PutSalesforceObjectIT,
	!FetchDropboxIT,
	!PutDropboxIT,
	!ListDropboxIT,
	!FetchBoxFileIT,
	!PutBoxFileIT,
	!ListBoxFileIT,
	!JsonConfigBasedBoxClientServiceIT,
	!QueryIoTDBIT,
	!PutIoTDBRecordIT,
	!StandardProcessSessionIT#testCloneThenWriteCountsClaimReferencesProperly,
	!StandardProcessSessionIT#testUpdateFlowFileRepoFailsOnSessionCommit,
	!JMSPublisherConsumerIT#testMultipleThreads,
	!ITRedisDistributedMapCacheClientService#testSentinelRedisWithAuthentication,
	!ITRedisDistributedMapCacheClientService#testSentinelRedis,
	!SnowflakePipeIT,
	!SecureFileIT,
	!SecureProxyIT#testAccessStatus,
	!SecureProxyIT#testAccessStatusAsProxiedAdmin,
	!SecureProxyIT#testAccessStatusUsingRegistryClient,
	!SecureNiFiRegistryClientIT#testGetAccessStatus,
	!SecureNiFiRegistryClientIT#testTenantsClientGroups,
	!SecureNiFiRegistryClientIT#testTenantsClientUsers,
	!SecureNiFiRegistryClientIT#testCrudOperations,
	!SecureDatabaseIT,
	!ITConnectionAccessControl,
	!ITLabelAccessControl,
	!ITOutputPortAccessControl,
	!ITProcessorAccessControl,
	!ITFlowAccessControl,
	!ITFunnelAccessControl,
	!ITInputPortAccessControl
	!ITCountersAccessControl,
	!ITAllowProxiedAnonymousAccess,
	!ITPreventProxiedAnonymousAccess,
	!ITPreventDirectAnonymousAccess,
	!ITAllowDirectAnonymousAccess,
	!ITProcessGroupAccessControl

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	permissions:
	contents: read

	jobs:
	build_and_test:
	strategy:
	fail-fast: false
	matrix:
	# TODO: macos-latest doesn't come with Docker and Colima seems to have problems with Test Containers - see NIFI-12191
	os: [ ubuntu-latest ]
	version: [ 21 ]
	timeout-minutes: 120
	runs-on: ${{ matrix.os }}
	name: ${{ matrix.os }} Java ${{ matrix.version }}
	env:
	# default to Amazon Corretto JDK
	JAVA_DISTRIBUTION: corretto
	steps:
	# Zulu JDK appears to be more reliable for MacOS
	- name: Use Java distribution Zulu
	if: ${{ runner.os == 'macos' }}
	run: echo "JAVA_DISTRIBUTION=zulu" >> "$GITHUB_ENV"
	- name: Checkout Code
	uses: actions/checkout@v4
	- name: Set up Java ${{ env.JAVA_DISTRIBUTION }} ${{ matrix.version }}
	uses: actions/setup-java@v3
	with:
	distribution: ${{ env.JAVA_DISTRIBUTION }}
	java-version: ${{ matrix.version }}
	cache: 'maven'

	# many integration-tests use Docker to run Testcontainers; MacOS doesn't come with Docker, so we need to install it
	- name: Set up Docker
	if: ${{ runner.os == 'macos' }}
	run: \|-
	/bin/bash -c "$(curl -fsSL https://github.com/raw/Homebrew/install/HEAD/install.sh)"
	brew install docker
	colima start

	if [ ! -e /var/run/docker.sock ]; then sudo ln -s ~/.colima/docker.sock /var/run/docker.sock; fi
	- name: Confirm Docker
	run: id && docker info

	- name: Build and Run Tests
	env:
	MAVEN_OPTS: >-
	${{ env.DEFAULT_MAVEN_OPTS }}
	run: >
	${{ env.MAVEN_COMMAND }}
	${{ env.MAVEN_BUILD_ARGUMENTS }}
	${{ env.MAVEN_BUILD_EXCLUDE_PROJECTS }}
	'-Dit.test=${{ env.MAVEN_FAILSAFE_SKIP_ITS }}'
	# - name: Run Failsafe Tests
	# env:
	# MAVEN_OPTS: >-
	# ${{ env.DEFAULT_MAVEN_OPTS }}
	# run: >
	# ${{ env.MAVEN_COMMAND }}
	# ${{ env.MAVEN_FAILSAFE_RUN_ARGUMENTS }}
	# '-Dit.test=${{ env.MAVEN_FAILSAFE_SKIP_ITS }}'
	- name: Upload Troubleshooting Logs
	if: failure() \|\| cancelled()
	uses: actions/upload-artifact@v3
	with:
	name: ${{ matrix.os }}-${{ matrix.version }}-failsafe-logs
	path: \|
	/target/failsafe-reports//*.txt
	retention-days: 7

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

NIFI-12178 add integration-tests and docker-tests github actions #4

Workflow file

NIFI-12178 add integration-tests and docker-tests github actions #4

Jobs

Run details

Workflow file for this run