You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ma...@apache.org on 2022/11/29 12:14:34 UTC

[flink] branch release-1.15 updated: [FLINK-29092][Connectors/Test] Use one bucket assigner

This is an automated email from the ASF dual-hosted git repository.

mapohl pushed a commit to branch release-1.15
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.15 by this push:
     new 84ec72f2cad [FLINK-29092][Connectors/Test] Use one bucket assigner
84ec72f2cad is described below

commit 84ec72f2cade11251c263a1a367634521461d225
Author: kurt <di...@163.com>
AuthorDate: Mon Sep 5 17:43:03 2022 +0800

    [FLINK-29092][Connectors/Test] Use one bucket assigner
    
    Signed-off-by: Matthias Pohl <ma...@aiven.io>
---
 .../hadoop/bulk/HadoopPathBasedPartFileWriterTest.java     | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java
index 9290c5c64d0..b143e06ca25 100644
--- a/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java
+++ b/flink-formats/flink-hadoop-bulk/src/test/java/org/apache/flink/formats/hadoop/bulk/HadoopPathBasedPartFileWriterTest.java
@@ -23,7 +23,7 @@ import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.functions.sink.filesystem.HadoopPathBasedBulkFormatBuilder;
 import org.apache.flink.streaming.api.functions.sink.filesystem.TestStreamingFileSinkFactory;
-import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner;
+import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.BasePathBucketAssigner;
 import org.apache.flink.streaming.util.FiniteTestSource;
 import org.apache.flink.test.util.AbstractTestBase;
 
@@ -81,16 +81,18 @@ public class HadoopPathBasedPartFileWriterTest extends AbstractTestBase {
         env.setParallelism(1);
         env.enableCheckpointing(100);
 
+        // FiniteTestSource will generate two elements with a checkpoint trigger in between the two
+        // elements
         DataStream<String> stream =
                 env.addSource(new FiniteTestSource<>(data), TypeInformation.of(String.class));
         Configuration configuration = new Configuration();
-
+        // Elements from source are going to be assigned to one bucket
         HadoopPathBasedBulkFormatBuilder<String, String, ?> builder =
                 new HadoopPathBasedBulkFormatBuilder<>(
                         basePath,
                         new TestHadoopPathBasedBulkWriterFactory(),
                         configuration,
-                        new DateTimeBucketAssigner<>());
+                        new BasePathBucketAssigner<>());
         TestStreamingFileSinkFactory<String> streamingFileSinkFactory =
                 new TestStreamingFileSinkFactory<>();
         stream.addSink(streamingFileSinkFactory.createSink(builder, 1000));
@@ -104,11 +106,7 @@ public class HadoopPathBasedPartFileWriterTest extends AbstractTestBase {
     private void validateResult(List<String> expected, Configuration config, Path basePath)
             throws IOException {
         FileSystem fileSystem = FileSystem.get(basePath.toUri(), config);
-        FileStatus[] buckets = fileSystem.listStatus(basePath);
-        assertNotNull(buckets);
-        assertEquals(1, buckets.length);
-
-        FileStatus[] partFiles = fileSystem.listStatus(buckets[0].getPath());
+        FileStatus[] partFiles = fileSystem.listStatus(basePath);
         assertNotNull(partFiles);
         assertEquals(2, partFiles.length);