You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ma...@apache.org on 2024/03/25 16:07:09 UTC
(flink) branch release-1.19 updated (5ec4bf2f181 -> f82ff7c656d)
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a change to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
from 5ec4bf2f181 [FLINK-32513][core] Add predecessor caching
new 4fc36e9abaa [FLINK-21400][ci] Enables FileSink and Stateful stream job e2e test for the AdaptiveScheduler
new 00492630baa [FLINK-21450][test] Enables tests that were disabled for the AdaptiveScheduler
new 7d107966dbe [FLINK-21535][test] Adds proper comment to test methods that are disabled for the AdaptiveScheduler
new f82ff7c656d [FLINK-34409][ci] Enable any still disabled e2e tests for the AdaptiveScheduler
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../file/src/FileSourceTextLinesITCase.java | 2 -
flink-end-to-end-tests/run-nightly-tests.sh | 60 ++++++++++------------
.../flink/runtime/jobmaster/JobMasterTest.java | 4 +-
.../checkpointing/UnalignedCheckpointITCase.java | 3 --
.../checkpointing/UnalignedCheckpointTestBase.java | 12 ++++-
.../DefaultSchedulerLocalRecoveryITCase.java | 9 +++-
6 files changed, 48 insertions(+), 42 deletions(-)
(flink) 01/04: [FLINK-21400][ci] Enables FileSink and Stateful stream job e2e test for the AdaptiveScheduler
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4fc36e9abaa8cc2d0e01c1e389b449f563b87e8e
Author: Matthias Pohl <ma...@aiven.io>
AuthorDate: Wed Feb 7 12:40:43 2024 +0100
[FLINK-21400][ci] Enables FileSink and Stateful stream job e2e test for the AdaptiveScheduler
---
flink-end-to-end-tests/run-nightly-tests.sh | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index 57f4fa22f7b..b5000eb5d38 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -189,14 +189,12 @@ function run_group_2 {
run_test "Streaming SQL end-to-end test using planner with Scala version" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh scala-planner" "skip_check_exceptions"
run_test "Sql Jdbc Driver end-to-end test" "$END_TO_END_DIR/test-scripts/test_sql_jdbc_driver.sh" "skip_check_exceptions"
- if [[ ${PROFILE} != *"enable-adaptive-scheduler"* ]]; then # FLINK-21400
- run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh local StreamingFileSink" "skip_check_exceptions"
- run_test "Streaming File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh s3 StreamingFileSink" "skip_check_exceptions"
- run_test "New File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh local FileSink" "skip_check_exceptions"
- run_test "New File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh s3 FileSink" "skip_check_exceptions"
+ run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh local StreamingFileSink" "skip_check_exceptions"
+ run_test "Streaming File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh s3 StreamingFileSink" "skip_check_exceptions"
+ run_test "New File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh local FileSink" "skip_check_exceptions"
+ run_test "New File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_file_sink.sh s3 FileSink" "skip_check_exceptions"
- run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
- fi
+ run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
run_test "Netty shuffle direct memory consumption end-to-end test" "$END_TO_END_DIR/test-scripts/test_netty_shuffle_memory_control.sh"
(flink) 04/04: [FLINK-34409][ci] Enable any still disabled e2e tests for the AdaptiveScheduler
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
commit f82ff7c656d3eeb3e82b456d284639e59624a849
Author: Matthias Pohl <ma...@aiven.io>
AuthorDate: Wed Feb 7 16:34:19 2024 +0100
[FLINK-34409][ci] Enable any still disabled e2e tests for the AdaptiveScheduler
---
flink-end-to-end-tests/run-nightly-tests.sh | 46 ++++++++++++++---------------
1 file changed, 22 insertions(+), 24 deletions(-)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index 72146e38ecd..6fe96ca4661 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -125,30 +125,28 @@ function run_group_1 {
# Docker / Container / Kubernetes tests
################################################################################
- if [[ ${PROFILE} != *"enable-adaptive-scheduler"* ]]; then
- run_test "Wordcount on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_docker_embedded_job.sh dummy-fs"
-
- run_test "Run Kubernetes test" "$END_TO_END_DIR/test-scripts/test_kubernetes_embedded_job.sh"
- run_test "Run kubernetes session test (default input)" "$END_TO_END_DIR/test-scripts/test_kubernetes_session.sh"
- run_test "Run kubernetes session test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_kubernetes_session.sh dummy-fs"
- run_test "Run kubernetes application test" "$END_TO_END_DIR/test-scripts/test_kubernetes_application.sh"
- run_test "Run kubernetes application HA test" "$END_TO_END_DIR/test-scripts/test_kubernetes_application_ha.sh"
- run_test "Run Kubernetes IT test" "$END_TO_END_DIR/test-scripts/test_kubernetes_itcases.sh"
-
- run_test "Running Flink over NAT end-to-end test" "$END_TO_END_DIR/test-scripts/test_nat.sh" "skip_check_exceptions"
-
- if [[ `uname -i` != 'aarch64' ]]; then
- # Skip PyFlink e2e test, because MiniConda and Pyarrow which Pyflink depends doesn't support aarch64 currently.
- run_test "Run kubernetes pyflink application test" "$END_TO_END_DIR/test-scripts/test_kubernetes_pyflink_application.sh"
-
- # Hadoop YARN deosn't support aarch64 at this moment. See: https://issues.apache.org/jira/browse/HADOOP-16723
- # These tests are known to fail on JDK11. See FLINK-13719
- if [[ ${PROFILE} != *"jdk11"* ]]; then
- run_test "Running Kerberized YARN per-job on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_job_kerberos_docker.sh"
- run_test "Running Kerberized YARN per-job on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_job_kerberos_docker.sh dummy-fs"
- run_test "Running Kerberized YARN application on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_application_kerberos_docker.sh"
- run_test "Running Kerberized YARN application on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_application_kerberos_docker.sh dummy-fs"
- fi
+ run_test "Wordcount on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_docker_embedded_job.sh dummy-fs"
+
+ run_test "Run Kubernetes test" "$END_TO_END_DIR/test-scripts/test_kubernetes_embedded_job.sh"
+ run_test "Run kubernetes session test (default input)" "$END_TO_END_DIR/test-scripts/test_kubernetes_session.sh"
+ run_test "Run kubernetes session test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_kubernetes_session.sh dummy-fs"
+ run_test "Run kubernetes application test" "$END_TO_END_DIR/test-scripts/test_kubernetes_application.sh"
+ run_test "Run kubernetes application HA test" "$END_TO_END_DIR/test-scripts/test_kubernetes_application_ha.sh"
+ run_test "Run Kubernetes IT test" "$END_TO_END_DIR/test-scripts/test_kubernetes_itcases.sh"
+
+ run_test "Running Flink over NAT end-to-end test" "$END_TO_END_DIR/test-scripts/test_nat.sh" "skip_check_exceptions"
+
+ if [[ `uname -i` != 'aarch64' ]]; then
+ # Skip PyFlink e2e test, because MiniConda and Pyarrow which Pyflink depends doesn't support aarch64 currently.
+ run_test "Run kubernetes pyflink application test" "$END_TO_END_DIR/test-scripts/test_kubernetes_pyflink_application.sh"
+
+ # Hadoop YARN doesn't support aarch64 at this moment. See: https://issues.apache.org/jira/browse/HADOOP-16723
+ # These tests are known to fail on JDK11. See FLINK-13719
+ if [[ ${PROFILE} != *"jdk11"* ]]; then
+ run_test "Running Kerberized YARN per-job on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_job_kerberos_docker.sh"
+ run_test "Running Kerberized YARN per-job on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_job_kerberos_docker.sh dummy-fs"
+ run_test "Running Kerberized YARN application on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_application_kerberos_docker.sh"
+ run_test "Running Kerberized YARN application on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_application_kerberos_docker.sh dummy-fs"
fi
fi
(flink) 03/04: [FLINK-21535][test] Adds proper comment to test methods that are disabled for the AdaptiveScheduler
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7d107966dbe7e38e43680fabf3ffdfeaa71e8d3c
Author: Matthias Pohl <ma...@aiven.io>
AuthorDate: Wed Feb 7 15:59:11 2024 +0100
[FLINK-21535][test] Adds proper comment to test methods that are disabled for the AdaptiveScheduler
---
.../flink/test/checkpointing/UnalignedCheckpointITCase.java | 3 ---
.../test/checkpointing/UnalignedCheckpointTestBase.java | 12 ++++++++++--
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java
index effbbc52fc8..f649b8f7df3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointITCase.java
@@ -38,12 +38,10 @@ import org.apache.flink.streaming.api.datastream.DataStreamSink;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
-import org.apache.flink.testutils.junit.FailsWithAdaptiveScheduler;
import org.apache.flink.util.Collector;
import org.apache.commons.lang3.ArrayUtils;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -105,7 +103,6 @@ import static org.hamcrest.Matchers.equalTo;
* </ul>
*/
@RunWith(Parameterized.class)
-@Category(FailsWithAdaptiveScheduler.class) // FLINK-21689
public class UnalignedCheckpointITCase extends UnalignedCheckpointTestBase {
enum Topology implements DagCreator {
PIPELINE {
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointTestBase.java
index ea664da1772..e88f27fe1ce 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointTestBase.java
@@ -104,8 +104,16 @@ import static org.apache.flink.shaded.guava31.com.google.common.collect.Iterable
import static org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions.CHECKPOINTING_TIMEOUT;
import static org.apache.flink.util.Preconditions.checkState;
-/** Base class for tests related to unaligned checkpoints. */
-@Category(FailsWithAdaptiveScheduler.class) // FLINK-21689
+/**
+ * Base class for tests related to unaligned checkpoints.
+ *
+ * <p>This test base relies on restarting the subtasks within the scheduler to trigger a reset of
+ * the operators. The operator reset is counted in the LongSource. The job will terminate if the
+ * number of expected restarts is reached. The AdaptiveScheduler won't trigger the operator reset
+ * resulting in the test running forever. This is why this test suite is disabled for the {@link
+ * org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler}.
+ */
+@Category(FailsWithAdaptiveScheduler.class)
public abstract class UnalignedCheckpointTestBase extends TestLogger {
protected static final Logger LOG = LoggerFactory.getLogger(UnalignedCheckpointTestBase.class);
protected static final String NUM_INPUTS = "inputs_";
(flink) 02/04: [FLINK-21450][test] Enables tests that were disabled for the AdaptiveScheduler
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mapohl pushed a commit to branch release-1.19
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 00492630baa5cf041ea2cce2a3560f3e713bf57a
Author: Matthias Pohl <ma...@aiven.io>
AuthorDate: Wed Feb 7 15:47:39 2024 +0100
[FLINK-21450][test] Enables tests that were disabled for the AdaptiveScheduler
For a few tests, a proper explanation is added why the tests are still disabled
---
.../flink/connector/file/src/FileSourceTextLinesITCase.java | 2 --
flink-end-to-end-tests/run-nightly-tests.sh | 2 +-
.../java/org/apache/flink/runtime/jobmaster/JobMasterTest.java | 4 +++-
.../flink/test/runtime/DefaultSchedulerLocalRecoveryITCase.java | 9 +++++++--
4 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/FileSourceTextLinesITCase.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/FileSourceTextLinesITCase.java
index 08d53f21426..f0ff3fb5cb2 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/FileSourceTextLinesITCase.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/src/FileSourceTextLinesITCase.java
@@ -42,7 +42,6 @@ import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.util.function.FunctionWithException;
import org.apache.flink.util.function.ThrowingConsumer;
-import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.junit.jupiter.api.io.TempDir;
@@ -198,7 +197,6 @@ class FileSourceTextLinesITCase {
* record format (text lines) and restarts TaskManager.
*/
@Test
- @Tag("org.apache.flink.testutils.junit.FailsWithAdaptiveScheduler") // FLINK-21450
void testContinuousTextFileSourceWithTaskManagerFailover(@TempDir java.nio.file.Path tmpTestDir)
throws Exception {
// This test will kill TM, so we run it in a new cluster to avoid affecting other tests
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index b5000eb5d38..72146e38ecd 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -233,7 +233,7 @@ function run_group_2 {
# Sticky Scheduling
################################################################################
- if [[ ${PROFILE} != *"enable-adaptive-scheduler"* ]]; then #FLINK-21450
+ if [[ ${PROFILE} != *"enable-adaptive-scheduler"* ]]; then # FLINK-34416
run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 hashmap false false 100" "skip_check_exceptions"
run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 3 hashmap false true 100" "skip_check_exceptions"
run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks false false 100" "skip_check_exceptions"
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
index 5a84fb4f0e4..e8e63652283 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTest.java
@@ -1019,7 +1019,9 @@ class JobMasterTest {
* if this execution fails.
*/
@Test
- @Tag("org.apache.flink.testutils.junit.FailsWithAdaptiveScheduler") // FLINK-21450
+ // The AdaptiveScheduler doesn't support partial recovery but restarts all Executions in case of
+ // a local failure.
+ @Tag("org.apache.flink.testutils.junit.FailsWithAdaptiveScheduler")
void testRequestNextInputSplitWithLocalFailover() throws Exception {
configuration.set(
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/DefaultSchedulerLocalRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/DefaultSchedulerLocalRecoveryITCase.java
index 44db8abdec6..583e4341e65 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/DefaultSchedulerLocalRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/DefaultSchedulerLocalRecoveryITCase.java
@@ -60,13 +60,18 @@ public class DefaultSchedulerLocalRecoveryITCase extends TestLogger {
private static final long TIMEOUT = 10_000L;
@Test
- @Category(FailsWithAdaptiveScheduler.class) // FLINK-21450
+ // The AdaptiveScheduler doesn't update the ExecutionGraph but creates a new Execution during
+ // local recovery. Recovering can also lead to a change in parallelism which makes the
+ // executionHistory non-linear. The lack of a linear executionHistory prevents us from applying
+ // the same test for the AdaptiveScheduler.
+ @Category(FailsWithAdaptiveScheduler.class)
public void testLocalRecoveryFull() throws Exception {
testLocalRecoveryInternal("full");
}
@Test
- @Category(FailsWithAdaptiveScheduler.class) // FLINK-21450
+ // see comment in #testLocalRecoveryFull
+ @Category(FailsWithAdaptiveScheduler.class)
public void testLocalRecoveryRegion() throws Exception {
testLocalRecoveryInternal("region");
}