You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2020/11/30 15:15:55 UTC
[flink] 08/10: [refactor] Factor common testing code out of
FileSinkITBase
This is an automated email from the ASF dual-hosted git repository.
aljoscha pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 0528a21f7768c5c653bcbe03437c7ab9681dcf86
Author: Yun Gao <ga...@gmail.com>
AuthorDate: Thu Nov 26 17:46:59 2020 +0800
[refactor] Factor common testing code out of FileSinkITBase
We want to reuse this code for the new FileSink migration test as well.
---
flink-connectors/flink-connector-files/pom.xml | 15 +++
.../flink/connector/file/sink/FileSinkITBase.java | 93 +-------------
.../sink/utils/IntegerFileSinkTestDataUtils.java | 140 +++++++++++++++++++++
3 files changed, 159 insertions(+), 89 deletions(-)
diff --git a/flink-connectors/flink-connector-files/pom.xml b/flink-connectors/flink-connector-files/pom.xml
index 117e2c5..65a9027 100644
--- a/flink-connectors/flink-connector-files/pom.xml
+++ b/flink-connectors/flink-connector-files/pom.xml
@@ -92,4 +92,19 @@ under the License.
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkITBase.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkITBase.java
index 6cd09d6..4912f2c 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkITBase.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkITBase.java
@@ -18,17 +18,14 @@
package org.apache.flink.connector.file.sink;
-import org.apache.flink.api.common.serialization.Encoder;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.connector.file.sink.utils.IntegerFileSinkTestDataUtils;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.minicluster.MiniCluster;
import org.apache.flink.runtime.minicluster.MiniClusterConfiguration;
-import org.apache.flink.streaming.api.functions.sink.filesystem.BucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.PartFileInfo;
-import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.SimpleVersionedStringSerializer;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.CheckpointRollingPolicy;
import org.apache.flink.util.TestLogger;
@@ -37,22 +34,9 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runners.Parameterized;
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
/**
* The base class for the File Sink IT Case in different execution mode.
@@ -101,88 +85,19 @@ public abstract class FileSinkITBase extends TestLogger {
miniCluster.executeJobBlocking(jobGraph);
}
- checkResult(path);
+ IntegerFileSinkTestDataUtils.checkIntegerSequenceSinkOutput(path, NUM_RECORDS, NUM_BUCKETS, NUM_SOURCES);
}
protected abstract JobGraph createJobGraph(String path);
protected FileSink<Integer> createFileSink(String path) {
return FileSink
- .forRowFormat(new Path(path), new IntEncoder())
- .withBucketAssigner(new ModuloBucketAssigner())
+ .forRowFormat(new Path(path), new IntegerFileSinkTestDataUtils.IntEncoder())
+ .withBucketAssigner(new IntegerFileSinkTestDataUtils.ModuloBucketAssigner(NUM_BUCKETS))
.withRollingPolicy(new PartSizeAndCheckpointRollingPolicy(1024))
.build();
}
- private void checkResult(String path) throws Exception {
- File dir = new File(path);
- String[] subDirNames = dir.list();
- assertNotNull(subDirNames);
-
- Arrays.sort(subDirNames, Comparator.comparingInt(Integer::parseInt));
- assertEquals(NUM_BUCKETS, subDirNames.length);
- for (int i = 0; i < NUM_BUCKETS; ++i) {
- assertEquals(Integer.toString(i), subDirNames[i]);
-
- // now check its content
- File bucketDir = new File(path, subDirNames[i]);
- assertTrue(
- bucketDir.getAbsolutePath() + " Should be a existing directory",
- bucketDir.isDirectory());
-
- Map<Integer, Integer> counts = new HashMap<>();
- File[] files = bucketDir.listFiles(f -> !f.getName().startsWith("."));
- assertNotNull(files);
-
- for (File file : files) {
- assertTrue(file.isFile());
-
- try (DataInputStream dataInputStream = new DataInputStream(new FileInputStream(file))) {
- while (true) {
- int value = dataInputStream.readInt();
- counts.compute(value, (k, v) -> v == null ? 1 : v + 1);
- }
- } catch (EOFException e) {
- // End the reading
- }
- }
-
- int expectedCount = NUM_RECORDS / NUM_BUCKETS +
- (i < NUM_RECORDS % NUM_BUCKETS ? 1 : 0);
- assertEquals(expectedCount, counts.size());
-
- for (int j = i; j < NUM_RECORDS; j += NUM_BUCKETS) {
- assertEquals(
- "The record " + j + " should occur " + NUM_SOURCES + " times, " +
- " but only occurs " + counts.getOrDefault(j, 0) + "time",
- NUM_SOURCES,
- counts.getOrDefault(j, 0).intValue());
- }
- }
- }
-
- private static class IntEncoder implements Encoder<Integer> {
-
- @Override
- public void encode(Integer element, OutputStream stream) throws IOException {
- stream.write(ByteBuffer.allocate(4).putInt(element).array());
- stream.flush();
- }
- }
-
- private static class ModuloBucketAssigner implements BucketAssigner<Integer, String> {
-
- @Override
- public String getBucketId(Integer element, Context context) {
- return Integer.toString(element % NUM_BUCKETS);
- }
-
- @Override
- public SimpleVersionedSerializer<String> getSerializer() {
- return SimpleVersionedStringSerializer.INSTANCE;
- }
- }
-
private static class PartSizeAndCheckpointRollingPolicy
extends CheckpointRollingPolicy<Integer, String> {
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java
new file mode 100644
index 0000000..a0dab46
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.utils;
+
+import org.apache.flink.api.common.serialization.Encoder;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.streaming.api.functions.sink.filesystem.BucketAssigner;
+import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.SimpleVersionedStringSerializer;
+
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Utilities for file sinks that writes a sequence of continues integers into files starting from 0.
+ * The sinks expect multiple sources writing the same sequence onto the disk and the integers are
+ * assigned to different buckets according to modulo.
+ */
+public class IntegerFileSinkTestDataUtils {
+
+ /**
+ * Testing sink {@link Encoder} that writes integer with its binary representation.
+ */
+ public static class IntEncoder implements Encoder<Integer> {
+
+ @Override
+ public void encode(Integer element, OutputStream stream) throws IOException {
+ stream.write(ByteBuffer.allocate(4).putInt(element).array());
+ stream.flush();
+ }
+ }
+
+ /**
+ * Testing {@link BucketAssigner} that assigns integers according to modulo.
+ */
+ public static class ModuloBucketAssigner implements BucketAssigner<Integer, String> {
+
+ private final int numBuckets;
+
+ public ModuloBucketAssigner(int numBuckets) {
+ this.numBuckets = numBuckets;
+ }
+
+ @Override
+ public String getBucketId(Integer element, Context context) {
+ return Integer.toString(element % numBuckets);
+ }
+
+ @Override
+ public SimpleVersionedSerializer<String> getSerializer() {
+ return SimpleVersionedStringSerializer.INSTANCE;
+ }
+ }
+
+ /**
+ * Verifies the files written by the sink contains the expected integer sequences.
+ * The integers are partition into different buckets according to module, and each
+ * integer will be repeated by <tt>numSources</tt> times.
+ *
+ * @param path The directory to check.
+ * @param numRecords The total number of records.
+ * @param numBuckets The number of buckets to assign.
+ * @param numSources The parallelism of sources generating the sequences. Each integer will be
+ * repeat for <tt>numSources</tt> times.
+ */
+ public static void checkIntegerSequenceSinkOutput(String path, int numRecords, int numBuckets, int numSources) throws Exception {
+ File dir = new File(path);
+ String[] subDirNames = dir.list();
+ assertNotNull(subDirNames);
+
+ Arrays.sort(subDirNames, Comparator.comparingInt(Integer::parseInt));
+ assertEquals(numBuckets, subDirNames.length);
+ for (int i = 0; i < numBuckets; ++i) {
+ assertEquals(Integer.toString(i), subDirNames[i]);
+
+ // now check its content
+ File bucketDir = new File(path, subDirNames[i]);
+ assertTrue(
+ bucketDir.getAbsolutePath() + " Should be a existing directory",
+ bucketDir.isDirectory());
+
+ Map<Integer, Integer> counts = new HashMap<>();
+ File[] files = bucketDir.listFiles(f -> !f.getName().startsWith("."));
+ assertNotNull(files);
+
+ for (File file : files) {
+ assertTrue(file.isFile());
+
+ try (DataInputStream dataInputStream = new DataInputStream(new FileInputStream(file))) {
+ while (true) {
+ int value = dataInputStream.readInt();
+ counts.compute(value, (k, v) -> v == null ? 1 : v + 1);
+ }
+ } catch (EOFException e) {
+ // End the reading
+ }
+ }
+
+ int expectedCount = numRecords / numBuckets +
+ (i < numRecords % numBuckets ? 1 : 0);
+ assertEquals(expectedCount, counts.size());
+
+ for (int j = i; j < numRecords; j += numBuckets) {
+ assertEquals(
+ "The record " + j + " should occur " + numSources + " times, " +
+ " but only occurs " + counts.getOrDefault(j, 0) + "time",
+ numSources,
+ counts.getOrDefault(j, 0).intValue());
+ }
+ }
+ }
+}