Skip to content

Commit

Permalink
Merge branch 'master' into beam-yaml-proto-bp
Browse files Browse the repository at this point in the history
  • Loading branch information
ffernandez92 authored Oct 18, 2024
2 parents f713b3e + f5b7194 commit 527a7e5
Show file tree
Hide file tree
Showing 161 changed files with 4,754 additions and 679 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
},
"JavaTestProperties": {
"SUPPORTED_VERSIONS": ["8", "11", "17", "21"],
"FLINK_VERSIONS": ["1.15", "1.16", "1.17", "1.18"],
"FLINK_VERSIONS": ["1.17", "1.18", "1.19"],
"SPARK_VERSIONS": ["2", "3"]
},
"GoTestProperties": {
Expand Down
2 changes: 1 addition & 1 deletion .github/trigger_files/IO_Iceberg_Integration_Tests.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 3
"modification": 4
}
3 changes: 2 additions & 1 deletion .github/trigger_files/beam_PostCommit_Go_VR_Flink.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
"modification": 1,
"https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"https://github.com/apache/beam/pull/32648": "testing flink 1.19 support"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"https://github.com/apache/beam/pull/32648": "testing flink 1.19 support"
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
"https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
"https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
"https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test",
"https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{

"https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
}
2 changes: 1 addition & 1 deletion .github/trigger_files/beam_PostCommit_Python.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run.",
"modification": 3
"modification": 4
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@

{
"comment": "Modify this file in a trivial way to cause this test suite to run"
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
"modification": 2
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
"modification": 2
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
}
3 changes: 3 additions & 0 deletions .github/trigger_files/beam_PostCommit_XVR_Flink.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"https://github.com/apache/beam/pull/32648": "testing addition of Flink 1.19 support"
}
4 changes: 2 additions & 2 deletions .github/workflows/beam_LoadTests_Java_GBK_Smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ jobs:
arguments: |
--info \
-PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \
-Prunner=:runners:flink:1.15 \
-Prunner=:runners:flink:1.19 \
'-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Smoke_test_arguments_3 }}' \
- name: run GroupByKey load test Spark
uses: ./.github/actions/gradle-command-self-hosted-action
Expand All @@ -115,4 +115,4 @@ jobs:
arguments: |
-PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \
-Prunner=:runners:spark:3 \
'-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Smoke_test_arguments_4 }}'
'-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Smoke_test_arguments_4 }}'
4 changes: 2 additions & 2 deletions .github/workflows/beam_PostCommit_Java_Examples_Flink.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ jobs:
- name: run examplesIntegrationTest script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:flink:1.15:examplesIntegrationTest
gradle-command: :runners:flink:1.19:examplesIntegrationTest
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v4
if: ${{ !success() }}
Expand All @@ -93,4 +93,4 @@ jobs:
with:
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
files: '**/build/test-results/**/*.xml'
files: '**/build/test-results/**/*.xml'
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,10 @@ jobs:
with:
java-version: default
- name: Authenticate on GCP
uses: google-github-actions/auth@v2
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCP_SA_KEY }}
project_id: ${{ secrets.GCP_PROJECT_ID }}
token_format: 'access_token'
- name: Setup gcloud
uses: google-github-actions/setup-gcloud@v2
with:
project_id: ${{ secrets.GCP_PROJECT_ID }}
skip_install: true
- name: run scheduled javaPostcommitIOPerformanceTests script
if: github.event_name == 'schedule' #This ensures only scheduled runs publish metrics publicly by changing which exportTable is configured
uses: ./.github/actions/gradle-command-self-hosted-action
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,13 @@ jobs:
with:
gradle-command: :sdks:java:testing:nexmark:run
arguments: |
-Pnexmark.runner=:runners:flink:1.15 \
-Pnexmark.runner=:runners:flink:1.19 \
"${{ env.GRADLE_COMMAND_ARGUMENTS }} --streaming=${{ matrix.streaming }} --queryLanguage=${{ matrix.queryLanguage }}" \
- name: run PostCommit Java Nexmark Flink (${{ matrix.streaming }}) script
if: matrix.queryLanguage == 'none'
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:java:testing:nexmark:run
arguments: |
-Pnexmark.runner=:runners:flink:1.15 \
"${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}"
-Pnexmark.runner=:runners:flink:1.19 \
"${{ env.GRADLE_COMMAND_ARGUMENTS }}--streaming=${{ matrix.streaming }}"
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ jobs:
- name: run PostCommit Java Flink PortableValidatesRunner Streaming script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: runners:flink:1.15:job-server:validatesPortableRunnerStreaming
gradle-command: runners:flink:1.19:job-server:validatesPortableRunnerStreaming
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v4
if: ${{ !success() }}
Expand All @@ -90,4 +90,4 @@ jobs:
with:
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
files: '**/build/test-results/**/*.xml'
files: '**/build/test-results/**/*.xml'
2 changes: 1 addition & 1 deletion .github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,5 +101,5 @@ jobs:
with:
gradle-command: :sdks:java:testing:tpcds:run
arguments: |
-Ptpcds.runner=:runners:flink:1.18 \
-Ptpcds.runner=:runners:flink:1.19 \
"-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --queries=${{env.tpcdsQueriesArg}}" \
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ jobs:
- name: run validatesRunner script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:flink:1.18:validatesRunner
gradle-command: :runners:flink:1.19:validatesRunner
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v4
if: ${{ !success() }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,11 @@ jobs:
11
- name: run jar Java8 script
run: |
./gradlew :runners:flink:1.15:jar :runners:flink:1.15:testJar
./gradlew :runners:flink:1.19:jar :runners:flink:1.19:testJar
- name: run validatesRunner Java8 script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:flink:1.15:validatesRunner
gradle-command: :runners:flink:1.19:validatesRunner
arguments: |
-x shadowJar \
-x shadowTestJar \
Expand All @@ -109,4 +109,4 @@ jobs:
large_files: true
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
files: '**/build/test-results/**/*.xml'
files: '**/build/test-results/**/*.xml'
96 changes: 96 additions & 0 deletions .github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: PostCommit Python Xlang IO Direct

on:
schedule:
- cron: '30 5/6 * * *'
pull_request_target:
paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Xlang_IO_Direct.json']
workflow_dispatch:

#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
permissions:
actions: write
pull-requests: write
checks: write
contents: read
deployments: read
id-token: none
issues: write
discussions: read
packages: read
pages: read
repository-projects: read
security-events: read
statuses: read

# This allows a subsequently queued workflow run to interrupt previous runs
concurrency:
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
cancel-in-progress: true

env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}

jobs:
beam_PostCommit_Python_Xlang_IO_Direct:
if: |
github.event_name == 'workflow_dispatch' ||
github.event_name == 'pull_request_target' ||
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
github.event.comment.body == 'Run Python_Xlang_IO_Direct PostCommit'
runs-on: [self-hosted, ubuntu-20.04, main]
timeout-minutes: 100
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
job_name: ["beam_PostCommit_Python_Xlang_IO_Direct"]
job_phrase: ["Run Python_Xlang_IO_Direct PostCommit"]
steps:
- uses: actions/checkout@v4
- name: Setup repository
uses: ./.github/actions/setup-action
with:
comment_phrase: ${{ matrix.job_phrase }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
python-version: |
3.9
3.12
- name: run PostCommit Python Xlang IO Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :sdks:python:test-suites:direct:ioCrossLanguagePostCommit
arguments: -PuseWheelDistribution
- name: Archive Python Test Results
uses: actions/upload-artifact@v4
if: failure()
with:
name: Python Test Results
path: '**/pytest*.xml'
- name: Publish Python Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
if: always()
with:
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
files: '**/pytest*.xml'
4 changes: 2 additions & 2 deletions .github/workflows/beam_PostCommit_XVR_Flink.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
FlinkVersion: 1.15
FlinkVersion: 1.19

jobs:
beam_PostCommit_XVR_Flink:
Expand Down Expand Up @@ -110,4 +110,4 @@ jobs:
with:
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
files: '**/build/test-results/**/*.xml'
files: '**/build/test-results/**/*.xml'
4 changes: 2 additions & 2 deletions .github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ jobs:
- name: run validatesPortableRunnerBatch script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:flink:1.15:job-server:validatesPortableRunnerBatch
gradle-command: :runners:flink:1.19:job-server:validatesPortableRunnerBatch
env:
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH }}
- name: Archive JUnit Test Results
Expand All @@ -108,4 +108,4 @@ jobs:
with:
name: java-code-coverage-report
path: "**/build/test-results/**/*.xml"
# TODO: Investigate 'Max retries exceeded' issue with EnricoMi/publish-unit-test-result-action@v2.
# TODO: Investigate 'Max retries exceeded' issue with EnricoMi/publish-unit-test-result-action@v2.
4 changes: 2 additions & 2 deletions .github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ jobs:
- name: run PreCommit Java PVR Flink Docker script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
gradle-command: :runners:flink:1.15:job-server:validatesPortableRunnerDocker
gradle-command: :runners:flink:1.19:job-server:validatesPortableRunnerDocker
env:
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
- name: Archive JUnit Test Results
Expand All @@ -114,4 +114,4 @@ jobs:
with:
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
files: '**/build/test-results/**/*.xml'
files: '**/build/test-results/**/*.xml'
Loading

0 comments on commit 527a7e5

Please sign in to comment.