You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/08/11 12:21:07 UTC
(iotdb) 27/47: Pipe IT & Subscription IT: migrate workflows that are not in HighPerformanceMode to daily IT (#15971)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.5
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 59dc804ebdf8bc0bc53afbef60deae951fe1ca12
Author: VGalaxies <vg...@apache.org>
AuthorDate: Fri Jul 25 18:08:21 2025 +0800
Pipe IT & Subscription IT: migrate workflows that are not in HighPerformanceMode to daily IT (#15971)
* setup
* fixup! setup
(cherry picked from commit 253434481f0c84514eec46154480909382535148)
---
.github/workflows/daily-it.yml | 902 ++++++++++++++++++++++++++++++++++++++++-
.github/workflows/pipe-it.yml | 87 +---
2 files changed, 913 insertions(+), 76 deletions(-)
diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml
index c4f72ff0fd6..8f28c54e56a 100644
--- a/.github/workflows/daily-it.yml
+++ b/.github/workflows/daily-it.yml
@@ -80,4 +80,904 @@ jobs:
with:
name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }}
path: integration-test/target/cluster-logs
- retention-days: 3
\ No newline at end of file
+ retention-days: 3
+ PipeSingle:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [17]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+ os: [ ubuntu-latest ]
+ exclude:
+ - cluster1: LightWeightStandaloneMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: LightWeightStandaloneMode
+ cluster2: ScalableSingleNodeMode
+ - cluster1: ScalableSingleNodeMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: ScalableSingleNodeMode
+ cluster2: HighPerformanceMode
+ - cluster1: HighPerformanceMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: HighPerformanceMode
+ cluster2: HighPerformanceMode
+ - cluster1: PipeConsensusBatchMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: PipeConsensusBatchMode
+ cluster2: HighPerformanceMode
+ - cluster1: PipeConsensusStreamMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: PipeConsensusStreamMode
+ cluster2: HighPerformanceMode
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+ -pl integration-test \
+ -am -PMultiClusterIT1 \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ PipeDualTreeAutoBasic:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [17]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ os: [ ubuntu-latest ]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2DualTreeAutoBasic \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ PipeDualTreeAutoEnhanced:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [17]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+ os: [ ubuntu-latest ]
+ exclude:
+ - cluster1: LightWeightStandaloneMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: LightWeightStandaloneMode
+ cluster2: ScalableSingleNodeMode
+ - cluster1: ScalableSingleNodeMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: ScalableSingleNodeMode
+ cluster2: HighPerformanceMode
+ - cluster1: HighPerformanceMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: HighPerformanceMode
+ cluster2: HighPerformanceMode
+ - cluster1: PipeConsensusBatchMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: PipeConsensusBatchMode
+ cluster2: HighPerformanceMode
+ - cluster1: PipeConsensusStreamMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: PipeConsensusStreamMode
+ cluster2: HighPerformanceMode
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2DualTreeAutoEnhanced \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ PipeDualTreeManual:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [17]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+ os: [ ubuntu-latest ]
+ exclude:
+ - cluster1: LightWeightStandaloneMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: LightWeightStandaloneMode
+ cluster2: ScalableSingleNodeMode
+ - cluster1: ScalableSingleNodeMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: ScalableSingleNodeMode
+ cluster2: HighPerformanceMode
+ - cluster1: HighPerformanceMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: HighPerformanceMode
+ cluster2: HighPerformanceMode
+ - cluster1: PipeConsensusBatchMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: PipeConsensusBatchMode
+ cluster2: HighPerformanceMode
+ - cluster1: PipeConsensusStreamMode
+ cluster2: LightWeightStandaloneMode
+ - cluster1: PipeConsensusStreamMode
+ cluster2: HighPerformanceMode
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2DualTreeManual \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ SubscriptionTreeArchVerification:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [ 17 ]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
+ cluster2: [ ScalableSingleNodeMode ]
+ os: [ ubuntu-latest ]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2SubscriptionTreeArchVerification \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ SubscriptionTableArchVerification:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [ 17 ]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster1: [ ScalableSingleNodeMode ]
+ cluster2: [ ScalableSingleNodeMode ]
+ os: [ ubuntu-latest ]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2SubscriptionTableArchVerification \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ SubscriptionTreeRegressionConsumer:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [ 17 ]
+ # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
+ cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
+ cluster2: [ ScalableSingleNodeMode ]
+ os: [ ubuntu-latest ]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ SubscriptionTreeRegressionMisc:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [ 17 ]
+ # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
+ cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
+ cluster2: [ ScalableSingleNodeMode ]
+ os: [ ubuntu-latest ]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ PipeDualTableManualBasic:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [17]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ os: [ ubuntu-latest ]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2DualTableManualBasic \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
+ PipeDualTableManualEnhanced:
+ strategy:
+ fail-fast: false
+ max-parallel: 15
+ matrix:
+ java: [17]
+ # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
+ cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ os: [ ubuntu-latest ]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: Cache Maven packages
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2-
+ - name: Sleep for a random duration between 0 and 10000 milliseconds
+ run: |
+ sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
+ - name: IT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in client.yml
+ # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
+ run: |
+ retry() {
+ local -i max_attempts=3
+ local -i attempt=1
+ local -i retry_sleep=5
+ local test_output
+
+ while [ $attempt -le $max_attempts ]; do
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
+ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
+ -pl integration-test \
+ -am -PMultiClusterIT2DualTableManualEnhanced \
+ -ntp >> ~/run-tests-$attempt.log && return 0
+ test_output=$(cat ~/run-tests-$attempt.log)
+
+ echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
+ echo "$test_output"
+ echo "==================== END: ~/run-tests-$attempt.log ======================"
+
+ if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
+ echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
+ fi
+
+ if echo "$test_output" | grep -q "Could not transfer artifact"; then
+ if [ $attempt -lt $max_attempts ]; then
+ echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
+ sleep $retry_sleep
+ attempt=$((attempt + 1))
+ else
+ echo "Test failed after $max_attempts attempts due to artifact transfer issue."
+ echo "Treating this as a success because the issue is likely transient."
+ return 0
+ fi
+ elif [ $? -ne 0 ]; then
+ echo "Test failed with a different error."
+ return 1
+ else
+ echo "Tests passed"
+ return 0
+ fi
+ done
+ }
+ retry
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
+ path: integration-test/target/cluster-logs
+ retention-days: 30
\ No newline at end of file
diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml
index 2c1b03623d1..97e6fad1903 100644
--- a/.github/workflows/pipe-it.yml
+++ b/.github/workflows/pipe-it.yml
@@ -40,30 +40,9 @@ jobs:
matrix:
java: [17]
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+ cluster1: [HighPerformanceMode]
+ cluster2: [HighPerformanceMode]
os: [ ubuntu-latest ]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusBatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusBatchMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusStreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusStreamMode
- cluster2: HighPerformanceMode
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
@@ -145,7 +124,7 @@ jobs:
matrix:
java: [17]
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ cluster: [HighPerformanceMode]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps:
@@ -228,30 +207,9 @@ jobs:
matrix:
java: [17]
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+ cluster1: [HighPerformanceMode]
+ cluster2: [HighPerformanceMode]
os: [ ubuntu-latest ]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusBatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusBatchMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusStreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusStreamMode
- cluster2: HighPerformanceMode
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
@@ -333,30 +291,9 @@ jobs:
matrix:
java: [17]
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
+ cluster1: [HighPerformanceMode]
+ cluster2: [HighPerformanceMode]
os: [ ubuntu-latest ]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusBatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusBatchMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusStreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusStreamMode
- cluster2: HighPerformanceMode
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
@@ -438,7 +375,7 @@ jobs:
matrix:
java: [ 17 ]
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
+ cluster1: [ ScalableSingleNodeMode ]
cluster2: [ ScalableSingleNodeMode ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
@@ -606,7 +543,7 @@ jobs:
matrix:
java: [ 17 ]
# do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
- cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
+ cluster1: [ ScalableSingleNodeMode ]
cluster2: [ ScalableSingleNodeMode ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
@@ -690,7 +627,7 @@ jobs:
matrix:
java: [ 17 ]
# do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
- cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
+ cluster1: [ ScalableSingleNodeMode ]
cluster2: [ ScalableSingleNodeMode ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
@@ -774,7 +711,7 @@ jobs:
matrix:
java: [17]
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ cluster: [HighPerformanceMode]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps:
@@ -857,7 +794,7 @@ jobs:
matrix:
java: [17]
# StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
+ cluster: [HighPerformanceMode]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps: