You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2022/02/16 17:26:27 UTC
[flink] 05/06: [FLINK-25583][connectors/filesystem] Add compaction support for FileSink.
This is an automated email from the ASF dual-hosted git repository.
gaoyunhaii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 255d65613ad348f66b1043211f51ce7a56dfeac6
Author: Gen Luo <lu...@gmail.com>
AuthorDate: Wed Feb 16 18:34:42 2022 +0800
[FLINK-25583][connectors/filesystem] Add compaction support for FileSink.
---
.../apache/flink/connector/file/sink/FileSink.java | 169 ++++++-
.../file/sink/compactor/ConcatFileCompactor.java | 72 +++
.../file/sink/compactor/DecoderBasedReader.java | 96 ++++
.../file/sink/compactor/FileCompactStrategy.java | 112 +++++
.../file/sink/compactor/FileCompactor.java | 48 ++
.../sink/compactor/IdenticalFileCompactor.java | 45 ++
.../sink/compactor/InputFormatBasedReader.java | 78 +++
.../compactor/OutputStreamBasedFileCompactor.java | 53 ++
.../sink/compactor/RecordWiseFileCompactor.java | 84 ++++
.../file/sink/compactor/SimpleStringDecoder.java | 55 +++
.../compactor/operator/CompactCoordinator.java | 265 ++++++++++
.../operator/CompactCoordinatorFactory.java | 77 +++
.../operator/CompactCoordinatorStateHandler.java | 89 ++++
.../CompactCoordinatorStateHandlerFactory.java | 87 ++++
.../sink/compactor/operator/CompactService.java | 147 ++++++
.../sink/compactor/operator/CompactorOperator.java | 343 +++++++++++++
.../operator/CompactorOperatorFactory.java | 93 ++++
.../operator/CompactorOperatorStateHandler.java | 323 +++++++++++++
.../CompactorOperatorStateHandlerFactory.java | 84 ++++
.../sink/compactor/operator/CompactorRequest.java | 70 +++
.../operator/CompactorRequestSerializer.java | 98 ++++
.../operator/CompactorRequestTypeInfo.java | 120 +++++
...FileSinkCommittableSerializerMigrationTest.java | 4 +-
.../sink/compactor/AbstractCompactTestBase.java | 58 +++
.../sink/compactor/CompactCoordinatorTest.java | 449 +++++++++++++++++
.../file/sink/compactor/CompactorOperatorTest.java | 534 +++++++++++++++++++++
.../operator/CompactorRequestTypeInfoTest.java | 41 ++
.../file/sink/utils/FileSinkTestUtils.java | 80 ++-
.../sink/utils/IntegerFileSinkTestDataUtils.java | 26 +
.../apache/flink/util/CloseShieldOutputStream.java | 56 +++
.../SerializableSupplierWithException.java | 34 ++
.../flink/formats/avro/AvroWriterFactory.java | 37 +-
flink-tests/pom.xml | 8 +
33 files changed, 3886 insertions(+), 49 deletions(-)
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/FileSink.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/FileSink.java
index 566037f..e0f9da7 100644
--- a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/FileSink.java
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/FileSink.java
@@ -23,11 +23,21 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.BulkWriter;
import org.apache.flink.api.common.serialization.Encoder;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.connector.sink2.Committer;
import org.apache.flink.api.connector.sink2.StatefulSink;
import org.apache.flink.api.connector.sink2.StatefulSink.WithCompatibleState;
import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
+import org.apache.flink.api.java.typeutils.EitherTypeInfo;
import org.apache.flink.connector.file.sink.committer.FileCommitter;
+import org.apache.flink.connector.file.sink.compactor.FileCompactStrategy;
+import org.apache.flink.connector.file.sink.compactor.FileCompactor;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactCoordinatorFactory;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactCoordinatorStateHandlerFactory;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorOperatorFactory;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorOperatorStateHandlerFactory;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorRequest;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorRequestTypeInfo;
import org.apache.flink.connector.file.sink.writer.DefaultFileWriterBucketFactory;
import org.apache.flink.connector.file.sink.writer.FileWriter;
import org.apache.flink.connector.file.sink.writer.FileWriterBucketFactory;
@@ -36,6 +46,10 @@ import org.apache.flink.connector.file.sink.writer.FileWriterBucketStateSerializ
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.WithPreCommitTopology;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.functions.sink.filesystem.BucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter;
import org.apache.flink.streaming.api.functions.sink.filesystem.BulkBucketWriter;
@@ -46,6 +60,7 @@ import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.CheckpointRollingPolicy;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy;
+import org.apache.flink.types.Either;
import org.apache.flink.util.FlinkRuntimeException;
import java.io.IOException;
@@ -109,7 +124,8 @@ import static org.apache.flink.util.Preconditions.checkState;
public class FileSink<IN>
implements StatefulSink<IN, FileWriterBucketState>,
TwoPhaseCommittingSink<IN, FileSinkCommittable>,
- WithCompatibleState {
+ WithCompatibleState,
+ WithPreCommitTopology<IN, FileSinkCommittable> {
private final BucketsBuilder<IN, ? extends BucketsBuilder<IN, ?>> bucketsBuilder;
@@ -177,6 +193,74 @@ public class FileSink<IN>
basePath, bulkWriterFactory, new DateTimeBucketAssigner<>());
}
+ @Override
+ public DataStream<CommittableMessage<FileSinkCommittable>> addPreCommitTopology(
+ DataStream<CommittableMessage<FileSinkCommittable>> committableStream) {
+ FileCompactStrategy strategy = bucketsBuilder.getCompactStrategy();
+ if (strategy == null) {
+ // not enabled, handlers will be added to process the remaining states of the compact
+ // coordinator and the compactor operators.
+ SingleOutputStreamOperator<
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>>
+ coordinatorOp =
+ committableStream
+ .forward()
+ .transform(
+ "CompactorCoordinator",
+ new EitherTypeInfo<>(
+ committableStream.getType(),
+ new CompactorRequestTypeInfo(
+ bucketsBuilder
+ ::getCommittableSerializer)),
+ new CompactCoordinatorStateHandlerFactory(
+ bucketsBuilder::getCommittableSerializer))
+ .setParallelism(committableStream.getParallelism())
+ .uid("FileSinkCompactorCoordinator");
+
+ return coordinatorOp
+ .forward()
+ .transform(
+ "CompactorOperator",
+ committableStream.getType(),
+ new CompactorOperatorStateHandlerFactory(
+ bucketsBuilder::getCommittableSerializer,
+ bucketsBuilder::createBucketWriter))
+ .setParallelism(committableStream.getParallelism())
+ .uid("FileSinkCompactorOperator");
+ }
+
+ // explicitly rebalance here is required, or the partitioner will be forward, which is in
+ // fact the partitioner from the writers to the committers
+ SingleOutputStreamOperator<CompactorRequest> coordinatorOp =
+ committableStream
+ .rebalance()
+ .transform(
+ "CompactorCoordinator",
+ new CompactorRequestTypeInfo(
+ bucketsBuilder::getCommittableSerializer),
+ new CompactCoordinatorFactory(
+ strategy, bucketsBuilder::getCommittableSerializer))
+ .setParallelism(1)
+ .uid("FileSinkCompactorCoordinator");
+
+ // parallelism of the compactors is not configurable at present, since it must be identical
+ // to that of the committers, or the committable summary and the committables may be
+ // distributed to different committers, which will cause a failure
+ TypeInformation<CommittableMessage<FileSinkCommittable>> committableType =
+ committableStream.getType();
+ return coordinatorOp
+ .transform(
+ "CompactorOperator",
+ committableType,
+ new CompactorOperatorFactory(
+ strategy,
+ bucketsBuilder.getFileCompactor(),
+ bucketsBuilder::getCommittableSerializer,
+ bucketsBuilder::createBucketWriter))
+ .setParallelism(committableStream.getParallelism())
+ .uid("FileSinkCompactorOperator");
+ }
+
/** The base abstract class for the {@link RowFormatBuilder} and {@link BulkFormatBuilder}. */
@Internal
private abstract static class BucketsBuilder<IN, T extends BucketsBuilder<IN, T>>
@@ -204,6 +288,15 @@ public class FileSink<IN>
@Internal
abstract SimpleVersionedSerializer<FileSinkCommittable> getCommittableSerializer()
throws IOException;
+
+ @Internal
+ abstract FileCompactStrategy getCompactStrategy();
+
+ @Internal
+ abstract FileCompactor getFileCompactor();
+
+ @Internal
+ abstract BucketWriter<IN, String> createBucketWriter() throws IOException;
}
/** A builder for configuring the sink for row-wise encoding formats. */
@@ -226,6 +319,10 @@ public class FileSink<IN>
private OutputFileConfig outputFileConfig;
+ private FileCompactStrategy compactStrategy;
+
+ private FileCompactor fileCompactor;
+
protected RowFormatBuilder(
Path basePath, Encoder<IN> encoder, BucketAssigner<IN, String> bucketAssigner) {
this(
@@ -275,6 +372,12 @@ public class FileSink<IN>
return self();
}
+ public T enableCompact(final FileCompactStrategy strategy, final FileCompactor compactor) {
+ this.compactStrategy = checkNotNull(strategy);
+ this.fileCompactor = checkNotNull(compactor);
+ return self();
+ }
+
/** Creates the actual sink. */
public FileSink<IN> build() {
return new FileSink<>(this);
@@ -282,6 +385,19 @@ public class FileSink<IN>
@Override
FileWriter<IN> createWriter(InitContext context) throws IOException {
+ OutputFileConfig writerFileConfig;
+ if (compactStrategy == null) {
+ writerFileConfig = outputFileConfig;
+ } else {
+ // Compaction is enabled. We always commit before compacting, so the file written by
+ // writer should be hid.
+ writerFileConfig =
+ OutputFileConfig.builder()
+ .withPartPrefix("." + outputFileConfig.getPartPrefix())
+ .withPartSuffix(outputFileConfig.getPartSuffix())
+ .build();
+ }
+
return new FileWriter<>(
basePath,
context.metricGroup(),
@@ -289,7 +405,7 @@ public class FileSink<IN>
bucketFactory,
createBucketWriter(),
rollingPolicy,
- outputFileConfig,
+ writerFileConfig,
context.getProcessingTimeService(),
bucketCheckInterval);
}
@@ -300,6 +416,16 @@ public class FileSink<IN>
}
@Override
+ FileCompactStrategy getCompactStrategy() {
+ return compactStrategy;
+ }
+
+ @Override
+ FileCompactor getFileCompactor() {
+ return fileCompactor;
+ }
+
+ @Override
SimpleVersionedSerializer<FileWriterBucketState> getWriterStateSerializer()
throws IOException {
BucketWriter<IN, String> bucketWriter = createBucketWriter();
@@ -319,7 +445,7 @@ public class FileSink<IN>
bucketWriter.getProperties().getInProgressFileRecoverableSerializer());
}
- private BucketWriter<IN, String> createBucketWriter() throws IOException {
+ BucketWriter<IN, String> createBucketWriter() throws IOException {
return new RowWiseBucketWriter<>(
FileSystem.get(basePath.toUri()).createRecoverableWriter(), encoder);
}
@@ -357,6 +483,10 @@ public class FileSink<IN>
private OutputFileConfig outputFileConfig;
+ private FileCompactStrategy compactStrategy;
+
+ private FileCompactor fileCompactor;
+
protected BulkFormatBuilder(
Path basePath,
BulkWriter.Factory<IN> writerFactory,
@@ -424,6 +554,12 @@ public class FileSink<IN>
outputFileConfig);
}
+ public T enableCompact(final FileCompactStrategy strategy, final FileCompactor compactor) {
+ this.compactStrategy = checkNotNull(strategy);
+ this.fileCompactor = checkNotNull(compactor);
+ return self();
+ }
+
/** Creates the actual sink. */
public FileSink<IN> build() {
return new FileSink<>(this);
@@ -431,6 +567,19 @@ public class FileSink<IN>
@Override
FileWriter<IN> createWriter(InitContext context) throws IOException {
+ OutputFileConfig writerFileConfig;
+ if (compactStrategy == null) {
+ writerFileConfig = outputFileConfig;
+ } else {
+ // Compaction is enabled. We always commit before compacting, so the file written by
+ // writer should be hid.
+ writerFileConfig =
+ OutputFileConfig.builder()
+ .withPartPrefix("." + outputFileConfig.getPartPrefix())
+ .withPartSuffix(outputFileConfig.getPartSuffix())
+ .build();
+ }
+
return new FileWriter<>(
basePath,
context.metricGroup(),
@@ -438,7 +587,7 @@ public class FileSink<IN>
bucketFactory,
createBucketWriter(),
rollingPolicy,
- outputFileConfig,
+ writerFileConfig,
context.getProcessingTimeService(),
bucketCheckInterval);
}
@@ -449,6 +598,16 @@ public class FileSink<IN>
}
@Override
+ FileCompactStrategy getCompactStrategy() {
+ return compactStrategy;
+ }
+
+ @Override
+ FileCompactor getFileCompactor() {
+ return fileCompactor;
+ }
+
+ @Override
SimpleVersionedSerializer<FileWriterBucketState> getWriterStateSerializer()
throws IOException {
BucketWriter<IN, String> bucketWriter = createBucketWriter();
@@ -468,7 +627,7 @@ public class FileSink<IN>
bucketWriter.getProperties().getInProgressFileRecoverableSerializer());
}
- private BucketWriter<IN, String> createBucketWriter() throws IOException {
+ BucketWriter<IN, String> createBucketWriter() throws IOException {
return new BulkBucketWriter<>(
FileSystem.get(basePath.toUri()).createRecoverableWriter(), writerFactory);
}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/ConcatFileCompactor.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/ConcatFileCompactor.java
new file mode 100644
index 0000000..64fba99
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/ConcatFileCompactor.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.FileSystem;
+import org.apache.flink.core.fs.Path;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+
+/**
+ * A {@link OutputStreamBasedFileCompactor} implementation that simply concat the compacting files.
+ * The fileDelimiter will be added between neighbouring files if provided.
+ */
+@PublicEvolving
+public class ConcatFileCompactor extends OutputStreamBasedFileCompactor {
+
+ private static final int CHUNK_SIZE = 4 * 1024 * 1024;
+
+ private final byte[] fileDelimiter;
+
+ public ConcatFileCompactor() {
+ this(null);
+ }
+
+ public ConcatFileCompactor(@Nullable byte[] fileDelimiter) {
+ this.fileDelimiter = fileDelimiter;
+ }
+
+ @Override
+ protected void doCompact(List<Path> inputFiles, OutputStream outputStream) throws Exception {
+ FileSystem fs = inputFiles.get(0).getFileSystem();
+ for (Path input : inputFiles) {
+ try (FSDataInputStream inputStream = fs.open(input)) {
+ copy(inputStream, outputStream);
+ }
+ if (fileDelimiter != null) {
+ outputStream.write(fileDelimiter);
+ }
+ }
+ }
+
+ private void copy(InputStream in, OutputStream out) throws IOException {
+ byte[] buf = new byte[CHUNK_SIZE];
+ int length;
+ while ((length = in.read(buf)) > 0) {
+ out.write(buf, 0, length);
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/DecoderBasedReader.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/DecoderBasedReader.java
new file mode 100644
index 0000000..55f4942
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/DecoderBasedReader.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.core.fs.FSDataInputStream;
+import org.apache.flink.core.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A {@link RecordWiseFileCompactor.Reader} implementation that reads the file as an {@link
+ * FSDataInputStream} and decodes the record with the {@link Decoder}.
+ */
+@PublicEvolving
+public class DecoderBasedReader<T> implements RecordWiseFileCompactor.Reader<T> {
+ private final Decoder<T> decoder;
+
+ public DecoderBasedReader(Path path, Decoder<T> decoder) throws IOException {
+ this.decoder = checkNotNull(decoder);
+ InputStream input = path.getFileSystem().open(path);
+ this.decoder.open(input);
+ }
+
+ @Override
+ public T read() throws IOException {
+ return decoder.decodeNext();
+ }
+
+ @Override
+ public void close() throws Exception {
+ decoder.close();
+ }
+
+ /**
+ * A {@link Decoder} to decode the file content into the actual records.
+ *
+ * <p>A {@link Decoder} is generally the reverse of a {@link
+ * org.apache.flink.api.common.serialization.Encoder}.
+ *
+ * @param <T> Thy type of the records the reader is reading.
+ */
+ public interface Decoder<T> extends Serializable {
+
+ /** Prepares to start decoding the input stream. */
+ void open(InputStream input) throws IOException;
+
+ /**
+ * @return The next record that decoded from the opened input stream, or null if no more
+ * available.
+ */
+ T decodeNext() throws IOException;
+
+ /** Closes the open resources. The decoder is responsible to close the input stream. */
+ void close() throws IOException;
+
+ /** Factory to create {@link Decoder}. */
+ interface Factory<T> extends Serializable {
+ Decoder<T> create();
+ }
+ }
+
+ /** Factory for {@link DecoderBasedReader}. */
+ public static class Factory<T> implements RecordWiseFileCompactor.Reader.Factory<T> {
+ private final Decoder.Factory<T> decoderFactory;
+
+ public Factory(Decoder.Factory<T> decoderFactory) {
+ this.decoderFactory = decoderFactory;
+ }
+
+ @Override
+ public DecoderBasedReader<T> createFor(Path path) throws IOException {
+ return new DecoderBasedReader<>(path, decoderFactory.create());
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/FileCompactStrategy.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/FileCompactStrategy.java
new file mode 100644
index 0000000..64c4145
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/FileCompactStrategy.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.connector.file.sink.FileSink;
+
+import java.io.Serializable;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/** Strategy for compacting the files written in {@link FileSink} before committing. */
+@PublicEvolving
+public class FileCompactStrategy implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // Compaction triggering strategies.
+ private final long sizeThreshold;
+ private final int numCheckpointsBeforeCompaction;
+
+ // Compaction executing strategies.
+ private final int numCompactThreads;
+
+ private FileCompactStrategy(
+ long sizeThreshold, int numCheckpointsBeforeCompaction, int numCompactThreads) {
+ this.sizeThreshold = sizeThreshold;
+ this.numCheckpointsBeforeCompaction = numCheckpointsBeforeCompaction;
+ this.numCompactThreads = numCompactThreads;
+ }
+
+ public long getSizeThreshold() {
+ return sizeThreshold;
+ }
+
+ public int getNumCheckpointsBeforeCompaction() {
+ return numCheckpointsBeforeCompaction;
+ }
+
+ public int getNumCompactThreads() {
+ return numCompactThreads;
+ }
+
+ /** Builder for {@link FileCompactStrategy}. */
+ public static class Builder {
+ private int numCheckpointsBeforeCompaction = -1;
+ private long sizeThreshold = -1;
+ private int numCompactThreads = -1;
+
+ public static FileCompactStrategy.Builder newBuilder() {
+ return new FileCompactStrategy.Builder();
+ }
+
+ /**
+ * Optional, compaction will be triggered when N checkpoints passed since the last
+ * triggering, -1 by default indicating no compaction on checkpoint.
+ */
+ public FileCompactStrategy.Builder enableCompactionOnCheckpoint(
+ int numCheckpointsBeforeCompaction) {
+ checkArgument(
+ numCheckpointsBeforeCompaction > 0,
+ "Number of checkpoints before compaction should be more than 0.");
+ this.numCheckpointsBeforeCompaction = numCheckpointsBeforeCompaction;
+ return this;
+ }
+
+ /**
+ * Optional, compaction will be triggered when the total size of compacting files reaches
+ * the threshold. -1 by default, indicating the size is unlimited.
+ */
+ public FileCompactStrategy.Builder setSizeThreshold(long sizeThreshold) {
+ this.sizeThreshold = sizeThreshold;
+ return this;
+ }
+
+ /** Optional, the count of compacting threads in a compactor operator, 1 by default. */
+ public FileCompactStrategy.Builder setNumCompactThreads(int numCompactThreads) {
+ checkArgument(numCompactThreads > 0, "Compact threads should be more than 0.");
+ this.numCompactThreads = numCompactThreads;
+ return this;
+ }
+
+ public FileCompactStrategy build() {
+ validate();
+ return new FileCompactStrategy(
+ sizeThreshold, numCheckpointsBeforeCompaction, numCompactThreads);
+ }
+
+ private void validate() {
+ if (sizeThreshold < 0 && numCheckpointsBeforeCompaction <= 0) {
+ throw new IllegalArgumentException(
+ "At least one trigger condition must be configured.");
+ }
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/FileCompactor.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/FileCompactor.java
new file mode 100644
index 0000000..b737fd3
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/FileCompactor.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * The {@link FileCompactor} is responsible for compacting files into one file.
+ *
+ * <p>The {@link FileCompactor} should declare which type of {@link CompactingFileWriter} is
+ * required, and invoke the writer correspondingly.
+ */
+@PublicEvolving
+public interface FileCompactor extends Serializable {
+
+ /** @return the {@link CompactingFileWriter} type the compactor will use. */
+ CompactingFileWriter.Type getWriterType();
+
+ /**
+ * Compact the given files into one file.
+ *
+ * @param inputFiles the files to be compacted.
+ * @param writer the writer to write the compacted file.
+ * @throws Exception Thrown if an exception occurs during the compacting.
+ */
+ void compact(List<Path> inputFiles, CompactingFileWriter writer) throws Exception;
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/IdenticalFileCompactor.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/IdenticalFileCompactor.java
new file mode 100644
index 0000000..8e6e4a1
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/IdenticalFileCompactor.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter;
+
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * A simple {@link OutputStreamBasedFileCompactor} implementation that directly copy the content of
+ * the only input file to the output.
+ */
+@Internal
+public class IdenticalFileCompactor extends ConcatFileCompactor {
+
+ public IdenticalFileCompactor() {
+ super();
+ }
+
+ @Override
+ public void compact(List<Path> inputFiles, CompactingFileWriter writer) throws Exception {
+ checkState(inputFiles.size() == 1, "IdenticalFileCompactor can only copy one input file");
+ super.compact(inputFiles, writer);
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/InputFormatBasedReader.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/InputFormatBasedReader.java
new file mode 100644
index 0000000..215f6bd
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/InputFormatBasedReader.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.core.fs.FileInputSplit;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.util.function.SerializableSupplierWithException;
+
+import java.io.IOException;
+
+/**
+ * A {@link RecordWiseFileCompactor.Reader} implementation that reads the file using the {@link
+ * FileInputFormat}.
+ */
+@PublicEvolving
+public class InputFormatBasedReader<T> implements RecordWiseFileCompactor.Reader<T> {
+ private final Path path;
+ private final FileInputFormat<T> inputFormat;
+
+ public InputFormatBasedReader(Path path, FileInputFormat<T> inputFormat) throws IOException {
+ this.path = path;
+ this.inputFormat = inputFormat;
+ open();
+ }
+
+ private void open() throws IOException {
+ long len = path.getFileSystem().getFileStatus(path).getLen();
+ inputFormat.open(new FileInputSplit(0, path, 0, len, null));
+ }
+
+ @Override
+ public T read() throws IOException {
+ if (inputFormat.reachedEnd()) {
+ return null;
+ }
+ return inputFormat.nextRecord(null);
+ }
+
+ @Override
+ public void close() throws IOException {
+ inputFormat.close();
+ }
+
+ /** Factory for {@link InputFormatBasedReader}. */
+ public static class Factory<T> implements RecordWiseFileCompactor.Reader.Factory<T> {
+ private final SerializableSupplierWithException<FileInputFormat<T>, IOException>
+ inputFormatFactory;
+
+ public Factory(
+ SerializableSupplierWithException<FileInputFormat<T>, IOException>
+ inputFormatFactory) {
+ this.inputFormatFactory = inputFormatFactory;
+ }
+
+ @Override
+ public InputFormatBasedReader<T> createFor(Path path) throws IOException {
+ return new InputFormatBasedReader<>(path, inputFormatFactory.get());
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/OutputStreamBasedFileCompactor.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/OutputStreamBasedFileCompactor.java
new file mode 100644
index 0000000..2ac3805
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/OutputStreamBasedFileCompactor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter;
+import org.apache.flink.util.CloseShieldOutputStream;
+
+import java.io.OutputStream;
+import java.util.List;
+
+/**
+ * Base class for {@link FileCompactor} implementations that use the {@link
+ * OutputStreamBasedCompactingFileWriter}.
+ */
+@PublicEvolving
+public abstract class OutputStreamBasedFileCompactor implements FileCompactor {
+ @Override
+ public final CompactingFileWriter.Type getWriterType() {
+ return CompactingFileWriter.Type.OUTPUT_STREAM;
+ }
+
+ @Override
+ public void compact(List<Path> inputFiles, CompactingFileWriter writer) throws Exception {
+ // The outputStream returned by OutputStreamBasedCompactingFileWriter#asOutputStream should
+ // not be closed here.
+ CloseShieldOutputStream outputStream =
+ new CloseShieldOutputStream(
+ ((OutputStreamBasedCompactingFileWriter) writer).asOutputStream());
+ doCompact(inputFiles, outputStream);
+ }
+
+ protected abstract void doCompact(List<Path> inputFiles, OutputStream outputStream)
+ throws Exception;
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/RecordWiseFileCompactor.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/RecordWiseFileCompactor.java
new file mode 100644
index 0000000..3244961
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/RecordWiseFileCompactor.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.RecordWiseCompactingFileWriter;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * A {@link FileCompactor} implementation that reads input files with a {@link Reader} and writes
+ * with the {@link RecordWiseCompactingFileWriter}.
+ */
+@PublicEvolving
+public class RecordWiseFileCompactor<IN> implements FileCompactor {
+ private final Reader.Factory<IN> readerFactory;
+
+ public RecordWiseFileCompactor(Reader.Factory<IN> readerFactory) {
+ this.readerFactory = readerFactory;
+ }
+
+ @Override
+ public final CompactingFileWriter.Type getWriterType() {
+ return CompactingFileWriter.Type.RECORD_WISE;
+ }
+
+ @Override
+ public void compact(List<Path> inputFiles, CompactingFileWriter writer) throws Exception {
+ RecordWiseCompactingFileWriter<IN> recordWriter =
+ (RecordWiseCompactingFileWriter<IN>) writer;
+ for (Path input : inputFiles) {
+ try (Reader<IN> reader = readerFactory.createFor(input)) {
+ IN elem;
+ while ((elem = reader.read()) != null) {
+ recordWriter.write(elem);
+ }
+ }
+ }
+ }
+
+ /**
+ * The reader that reads record from the compacting files.
+ *
+ * @param <T> Thy type of the records that is read.
+ */
+ public interface Reader<T> extends AutoCloseable {
+
+ /** @return The next record, or null if no more available. */
+ T read() throws IOException;
+
+ /**
+ * Factory for {@link Reader}.
+ *
+ * @param <T> Thy type of the records that is read.
+ */
+ interface Factory<T> extends Serializable {
+ /**
+ * @return A reader that reads elements from the given file.
+ * @throws IOException Thrown if an I/O error occurs when opening the file.
+ */
+ Reader<T> createFor(Path path) throws IOException;
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/SimpleStringDecoder.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/SimpleStringDecoder.java
new file mode 100644
index 0000000..49661ba7
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/SimpleStringDecoder.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.connector.file.sink.compactor.DecoderBasedReader.Decoder;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * A sink {@link Decoder} that reads data encoded by the {@link
+ * org.apache.flink.api.common.serialization.SimpleStringEncoder} only for compaction. The original
+ * input type is missing, but it's enough to read string contents for writing the compacted file.
+ */
+@PublicEvolving
+public class SimpleStringDecoder implements Decoder<String> {
+
+ private BufferedReader reader;
+
+ @Override
+ public void open(InputStream input) throws IOException {
+ this.reader = new BufferedReader(new InputStreamReader(input));
+ }
+
+ @Override
+ public String decodeNext() throws IOException {
+ String nextLine = reader.readLine();
+ // String read will be write directly to the compacted file, so the '\n' should be appended.
+ return nextLine == null ? null : (nextLine + '\n');
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinator.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinator.java
new file mode 100644
index 0000000..23437a5
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinator.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
+import org.apache.flink.connector.file.sink.FileSink;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.compactor.FileCompactStrategy;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.BoundedOneInput;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.util.SimpleVersionedListState;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/**
+ * Coordinator that coordinates file compaction for the {@link FileSink}.
+ *
+ * <p>All committable emitted from the writers are collected and packed into {@link
+ * CompactorRequest}s. The {@link FileCompactStrategy} defines when the requests can be fired. When
+ * a firing condition is met, the requests will be sent to the {@link CompactorOperator}.
+ *
+ * <p>The {@link CompactCoordinator} stores the non-fired committable as its state, and may emit a
+ * request at any time. A {@link CompactorOperator} must ensure that the ownership of the
+ * committable in a compact request is successfully handed from the coordinator, before it can
+ * actually perform the compaction.
+ */
+@Internal
+public class CompactCoordinator extends AbstractStreamOperator<CompactorRequest>
+ implements OneInputStreamOperator<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>,
+ BoundedOneInput {
+
+ static final ListStateDescriptor<byte[]> REMAINING_COMMITTABLE_RAW_STATES_DESC =
+ new ListStateDescriptor<>(
+ "remaining_compact_commit_raw_state", BytePrimitiveArraySerializer.INSTANCE);
+
+ private final FileCompactStrategy strategy;
+ private final SimpleVersionedSerializer<FileSinkCommittable> committableSerializer;
+
+ private final Map<String, CompactorRequest> packingRequests = new HashMap<>();
+ private final Map<String, CompactTrigger> triggers = new HashMap<>();
+
+ private ListState<FileSinkCommittable> remainingCommittableState;
+
+ public CompactCoordinator(
+ FileCompactStrategy strategy,
+ SimpleVersionedSerializer<FileSinkCommittable> committableSerializer) {
+ this.strategy = strategy;
+ this.committableSerializer = checkNotNull(committableSerializer);
+ }
+
+ @Override
+ public void processElement(StreamRecord<CommittableMessage<FileSinkCommittable>> element)
+ throws Exception {
+ CommittableMessage<FileSinkCommittable> message = element.getValue();
+ if (message instanceof CommittableWithLineage) {
+ FileSinkCommittable committable =
+ ((CommittableWithLineage<FileSinkCommittable>) element.getValue())
+ .getCommittable();
+ if (packAndTrigger(committable)) {
+ fireAndPurge(committable.getBucketId());
+ }
+ }
+ // or message instanceof CommittableSummary
+ // info in CommittableSummary is not necessary for compacting at present, ignore it
+ }
+
+ private boolean packAndTrigger(FileSinkCommittable committable) {
+ String bucketId = committable.getBucketId();
+ CompactorRequest bucketRequest =
+ packingRequests.computeIfAbsent(bucketId, CompactorRequest::new);
+ if (committable.hasInProgressFileToCleanup() || committable.hasCompactedFileToCleanup()) {
+ checkState(!committable.hasPendingFile());
+ // cleanup request, pass through directly
+ bucketRequest.addToPassthrough(committable);
+ return false;
+ }
+
+ if (!committable.hasPendingFile()) {
+ throw new RuntimeException("Committable to compact has no content.");
+ }
+
+ CompactTrigger trigger =
+ triggers.computeIfAbsent(bucketId, id -> new CompactTrigger(strategy));
+ CompactTriggerResult triggerResult = trigger.onElement(committable);
+ switch (triggerResult) {
+ case PASS_THROUGH:
+ bucketRequest.addToPassthrough(committable);
+ return false;
+ case CONTINUE:
+ bucketRequest.addToCompact(committable);
+ return false;
+ case FIRE_AND_PURGE:
+ bucketRequest.addToCompact(committable);
+ return true;
+ default:
+ throw new RuntimeException("Unexpected trigger result:" + triggerResult);
+ }
+ }
+
+ private void fireAndPurge(String bucketId) {
+ triggers.remove(bucketId);
+ CompactorRequest request = packingRequests.remove(bucketId);
+ if (request != null) {
+ output.collect(new StreamRecord<>(request));
+ }
+ }
+
+ @Override
+ public void endInput() throws Exception {
+ // emit all requests remained
+ for (CompactorRequest request : packingRequests.values()) {
+ output.collect(new StreamRecord<>(request));
+ }
+ packingRequests.clear();
+ triggers.clear();
+ }
+
+ @Override
+ public void prepareSnapshotPreBarrier(long checkpointId) throws Exception {
+ super.prepareSnapshotPreBarrier(checkpointId);
+
+ // trigger on checkpoint
+ List<String> bucketsToFire = new ArrayList<>(triggers.size());
+ for (Map.Entry<String, CompactTrigger> e : triggers.entrySet()) {
+ String bucketId = e.getKey();
+ CompactTrigger trigger = e.getValue();
+ if (trigger.onCheckpoint(checkpointId) == CompactTriggerResult.FIRE_AND_PURGE) {
+ bucketsToFire.add(bucketId);
+ }
+ }
+ bucketsToFire.forEach(this::fireAndPurge);
+ }
+
+ @Override
+ public void snapshotState(StateSnapshotContext context) throws Exception {
+ super.snapshotState(context);
+
+ List<FileSinkCommittable> remainingCommittable =
+ packingRequests.values().stream()
+ .flatMap(r -> r.getCommittableToCompact().stream())
+ .collect(Collectors.toList());
+ packingRequests.values().stream()
+ .flatMap(r -> r.getCommittableToPassthrough().stream())
+ .forEach(remainingCommittable::add);
+ remainingCommittableState.update(remainingCommittable);
+
+ // triggers will be recomputed when restoring so it's not necessary to store
+ }
+
+ @Override
+ public void initializeState(StateInitializationContext context) throws Exception {
+ super.initializeState(context);
+
+ remainingCommittableState =
+ new SimpleVersionedListState<>(
+ context.getOperatorStateStore()
+ .getListState(REMAINING_COMMITTABLE_RAW_STATES_DESC),
+ committableSerializer);
+
+ Iterable<FileSinkCommittable> stateRemaining = remainingCommittableState.get();
+ if (stateRemaining != null) {
+ for (FileSinkCommittable committable : stateRemaining) {
+ // restore and redistribute
+ if (packAndTrigger(committable)) {
+ fireAndPurge(committable.getBucketId());
+ }
+ }
+ }
+ }
+
+ enum CompactTriggerResult {
+ CONTINUE,
+ FIRE_AND_PURGE,
+ PASS_THROUGH
+ }
+
+ private static class CompactTrigger {
+ private final long threshold;
+ private final int numCheckpointsBeforeCompaction;
+
+ private long size;
+ private long triggeredCpId = -1;
+
+ CompactTrigger(FileCompactStrategy strategy) {
+ this.threshold = strategy.getSizeThreshold();
+ this.numCheckpointsBeforeCompaction = strategy.getNumCheckpointsBeforeCompaction();
+ }
+
+ public CompactTriggerResult onElement(FileSinkCommittable committable) {
+ if (threshold < 0) {
+ return CompactTriggerResult.CONTINUE;
+ }
+
+ PendingFileRecoverable file = committable.getPendingFile();
+ if (file == null) {
+ return CompactTriggerResult.PASS_THROUGH;
+ }
+
+ if (file.getPath() == null || !file.getPath().getName().startsWith(".")) {
+ // the file may be written with writer of elder version, or
+ // the file will be visible once committed, so it can not be compacted.
+ // pass through, add to results, do not add to compacting files
+ return CompactTriggerResult.PASS_THROUGH;
+ }
+
+ long curSize = file.getSize();
+ if (curSize < 0) {
+ // unrecognized committable, can not compact, pass through directly
+ return CompactTriggerResult.PASS_THROUGH;
+ }
+
+ size += curSize;
+ return size >= threshold
+ ? CompactTriggerResult.FIRE_AND_PURGE
+ : CompactTriggerResult.CONTINUE;
+ }
+
+ public CompactTriggerResult onCheckpoint(long checkpointId) {
+ if (numCheckpointsBeforeCompaction < 0) {
+ return CompactTriggerResult.CONTINUE;
+ }
+ if (triggeredCpId < 0) {
+ triggeredCpId = checkpointId - 1;
+ }
+ return checkpointId - triggeredCpId >= numCheckpointsBeforeCompaction
+ ? CompactTriggerResult.FIRE_AND_PURGE
+ : CompactTriggerResult.CONTINUE;
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorFactory.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorFactory.java
new file mode 100644
index 0000000..f06b09e
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorFactory.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.compactor.FileCompactStrategy;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+import org.apache.flink.util.function.SerializableSupplierWithException;
+
+import java.io.IOException;
+
+/** Factory for {@link CompactCoordinator}. */
+@Internal
+public class CompactCoordinatorFactory extends AbstractStreamOperatorFactory<CompactorRequest>
+ implements OneInputStreamOperatorFactory<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest> {
+
+ private final FileCompactStrategy strategy;
+ private final SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier;
+
+ public CompactCoordinatorFactory(
+ FileCompactStrategy strategy,
+ SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier) {
+ this.strategy = strategy;
+ this.committableSerializerSupplier = committableSerializerSupplier;
+ }
+
+ @Override
+ public <T extends StreamOperator<CompactorRequest>> T createStreamOperator(
+ StreamOperatorParameters<CompactorRequest> parameters) {
+ try {
+ final CompactCoordinator compactOperator =
+ new CompactCoordinator(strategy, committableSerializerSupplier.get());
+ compactOperator.setup(
+ parameters.getContainingTask(),
+ parameters.getStreamConfig(),
+ parameters.getOutput());
+ return (T) compactOperator;
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Cannot create commit operator for "
+ + parameters.getStreamConfig().getOperatorName(),
+ e);
+ }
+ }
+
+ @Override
+ public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {
+ return CompactCoordinator.class;
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorStateHandler.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorStateHandler.java
new file mode 100644
index 0000000..2b147ae
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorStateHandler.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.api.common.state.CheckpointListener;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.BoundedOneInput;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.util.SimpleVersionedListState;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.types.Either;
+
+import static org.apache.flink.connector.file.sink.compactor.operator.CompactCoordinator.REMAINING_COMMITTABLE_RAW_STATES_DESC;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/** Handler that processes the state of {@link CompactCoordinator} when compaction is disabled. */
+public class CompactCoordinatorStateHandler
+ extends AbstractStreamOperator<
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>>
+ implements OneInputStreamOperator<
+ CommittableMessage<FileSinkCommittable>,
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>>,
+ BoundedOneInput,
+ CheckpointListener {
+
+ private final SimpleVersionedSerializer<FileSinkCommittable> committableSerializer;
+
+ public CompactCoordinatorStateHandler(
+ SimpleVersionedSerializer<FileSinkCommittable> committableSerializer) {
+ this.committableSerializer = checkNotNull(committableSerializer);
+ }
+
+ @Override
+ public void processElement(StreamRecord<CommittableMessage<FileSinkCommittable>> element)
+ throws Exception {
+ output.collect(new StreamRecord<>(Either.Left(element.getValue())));
+ }
+
+ @Override
+ public void endInput() throws Exception {}
+
+ @Override
+ public void initializeState(StateInitializationContext context) throws Exception {
+ super.initializeState(context);
+
+ ListState<FileSinkCommittable> remainingCommittableState =
+ new SimpleVersionedListState<>(
+ context.getOperatorStateStore()
+ .getListState(REMAINING_COMMITTABLE_RAW_STATES_DESC),
+ committableSerializer);
+
+ Iterable<FileSinkCommittable> stateRemaining = remainingCommittableState.get();
+ if (stateRemaining != null) {
+ for (FileSinkCommittable committable : stateRemaining) {
+ // all committable should be wrapped with a single request, since multiple files
+ // compacting is not available now
+ String bucketId = committable.getBucketId();
+ CompactorRequest request = new CompactorRequest(bucketId);
+ request.addToCompact(committable);
+ output.collect(new StreamRecord<>(Either.Right(request)));
+ }
+ }
+
+ // Remaining committable should be all emitted, and the state can be cleared. From now on
+ // the operator is stateless, snapshotState is not necessary.
+ remainingCommittableState.clear();
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorStateHandlerFactory.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorStateHandlerFactory.java
new file mode 100644
index 0000000..d4ebe11
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactCoordinatorStateHandlerFactory.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+import org.apache.flink.types.Either;
+import org.apache.flink.util.function.SerializableSupplierWithException;
+
+import java.io.IOException;
+
+/** Factory for {@link CompactCoordinatorStateHandler}. */
+@Internal
+public class CompactCoordinatorStateHandlerFactory
+ extends AbstractStreamOperatorFactory<
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>>
+ implements OneInputStreamOperatorFactory<
+ CommittableMessage<FileSinkCommittable>,
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>> {
+
+ private final SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier;
+
+ public CompactCoordinatorStateHandlerFactory(
+ SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier) {
+ this.committableSerializerSupplier = committableSerializerSupplier;
+ }
+
+ @Override
+ public <
+ T extends
+ StreamOperator<
+ Either<
+ CommittableMessage<FileSinkCommittable>,
+ CompactorRequest>>>
+ T createStreamOperator(
+ StreamOperatorParameters<
+ Either<
+ CommittableMessage<FileSinkCommittable>,
+ CompactorRequest>>
+ parameters) {
+ try {
+ final CompactCoordinatorStateHandler handler =
+ new CompactCoordinatorStateHandler(committableSerializerSupplier.get());
+ handler.setup(
+ parameters.getContainingTask(),
+ parameters.getStreamConfig(),
+ parameters.getOutput());
+ return (T) handler;
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Cannot create commit operator for "
+ + parameters.getStreamConfig().getOperatorName(),
+ e);
+ }
+ }
+
+ @Override
+ public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {
+ return CompactCoordinator.class;
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactService.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactService.java
new file mode 100644
index 0000000..7e5c7f6
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactService.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.compactor.FileCompactor;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.util.Hardware;
+import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable;
+import org.apache.flink.util.concurrent.ExecutorThreadFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.flink.util.Preconditions.checkState;
+
+/** The asynchronous file compaction service. */
+@Internal
+public class CompactService {
+ private static final String COMPACTED_PREFIX = "compacted-";
+
+ private final int numCompactThreads;
+ private final FileCompactor fileCompactor;
+ private final BucketWriter<?, String> bucketWriter;
+
+ private transient ExecutorService compactService;
+
+ public CompactService(
+ int numCompactThreads,
+ FileCompactor fileCompactor,
+ BucketWriter<?, String> bucketWriter) {
+ this.numCompactThreads = numCompactThreads;
+ this.fileCompactor = fileCompactor;
+ this.bucketWriter = bucketWriter;
+ }
+
+ public void open() {
+ compactService =
+ Executors.newFixedThreadPool(
+ Math.max(1, Math.min(numCompactThreads, Hardware.getNumberCPUCores())),
+ new ExecutorThreadFactory("compact-executor"));
+ }
+
+ public void submit(
+ CompactorRequest request,
+ CompletableFuture<Iterable<FileSinkCommittable>> resultFuture) {
+ compactService.submit(
+ () -> {
+ try {
+ Iterable<FileSinkCommittable> result = compact(request);
+ resultFuture.complete(result);
+ } catch (Exception e) {
+ resultFuture.completeExceptionally(e);
+ }
+ });
+ }
+
+ public void close() {
+ if (compactService != null) {
+ compactService.shutdownNow();
+ }
+ }
+
+ private Iterable<FileSinkCommittable> compact(CompactorRequest request) throws Exception {
+ List<FileSinkCommittable> results = new ArrayList<>(request.getCommittableToPassthrough());
+
+ List<Path> compactingFiles = getCompactingPath(request, results);
+ if (compactingFiles.isEmpty()) {
+ return results;
+ }
+
+ Path targetPath = assembleCompactedFilePath(compactingFiles.get(0));
+ CompactingFileWriter compactingFileWriter =
+ bucketWriter.openNewCompactingFile(
+ fileCompactor.getWriterType(),
+ request.getBucketId(),
+ targetPath,
+ System.currentTimeMillis());
+ fileCompactor.compact(compactingFiles, compactingFileWriter);
+ PendingFileRecoverable compactedPendingFile = compactingFileWriter.closeForCommit();
+
+ FileSinkCommittable compacted =
+ new FileSinkCommittable(request.getBucketId(), compactedPendingFile);
+ results.add(compacted);
+ for (Path f : compactingFiles) {
+ // cleanup compacted files
+ results.add(new FileSinkCommittable(request.getBucketId(), f));
+ }
+
+ return results;
+ }
+
+ // results: side output pass through committable
+ private List<Path> getCompactingPath(
+ CompactorRequest request, List<FileSinkCommittable> results) throws IOException {
+ List<FileSinkCommittable> compactingCommittable = request.getCommittableToCompact();
+ List<Path> compactingFiles = new ArrayList<>();
+
+ for (FileSinkCommittable committable : compactingCommittable) {
+ PendingFileRecoverable pendingFile = committable.getPendingFile();
+ checkState(
+ pendingFile != null, "Illegal committable to compact, pending file is null.");
+
+ Path pendingPath = pendingFile.getPath();
+ checkState(
+ pendingPath != null && pendingPath.getName().startsWith("."),
+ "Illegal pending file to compact, path should start with . but is "
+ + pendingPath);
+
+ // commit the pending file and compact the committed file
+ bucketWriter.recoverPendingFile(pendingFile).commitAfterRecovery();
+ compactingFiles.add(pendingPath);
+ }
+ return compactingFiles;
+ }
+
+ private static Path assembleCompactedFilePath(Path uncompactedPath) {
+ String uncompactedName = uncompactedPath.getName();
+ if (uncompactedName.startsWith(".")) {
+ uncompactedName = uncompactedName.substring(1);
+ }
+ return new Path(uncompactedPath.getParent(), COMPACTED_PREFIX + uncompactedName);
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperator.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperator.java
new file mode 100644
index 0000000..33365b9
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperator.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.state.CheckpointListener;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.connector.file.sink.FileSink;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.compactor.FileCompactStrategy;
+import org.apache.flink.connector.file.sink.compactor.FileCompactor;
+import org.apache.flink.core.io.SimpleVersionedSerialization;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputSerializer;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableSummary;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.BoundedOneInput;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.util.SimpleVersionedListState;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * An operator that perform compaction for the {@link FileSink}.
+ *
+ * <p>Requests received from the {@link CompactCoordinator} will firstly be held in memory, and
+ * snapshot into the state of a checkpoint. When the checkpoint is successfully completed, all
+ * requests received before can be submitted. The results can be emitted at the next {@link
+ * #prepareSnapshotPreBarrier} invoking after the compaction is finished, to ensure that committers
+ * can receive only one CommittableSummary and the corresponding number of Committable for a single
+ * checkpoint.
+ */
+@Internal
+public class CompactorOperator
+ extends AbstractStreamOperator<CommittableMessage<FileSinkCommittable>>
+ implements OneInputStreamOperator<
+ CompactorRequest, CommittableMessage<FileSinkCommittable>>,
+ BoundedOneInput,
+ CheckpointListener {
+
+ private static final long SUBMITTED_ID = -1L;
+
+ static final ListStateDescriptor<byte[]> REMAINING_REQUESTS_RAW_STATES_DESC =
+ new ListStateDescriptor<>(
+ "remaining_requests_raw_state", BytePrimitiveArraySerializer.INSTANCE);
+
+ private final FileCompactStrategy strategy;
+ private final SimpleVersionedSerializer<FileSinkCommittable> committableSerializer;
+
+ private final FileCompactor fileCompactor;
+ private final BucketWriter<?, String> bucketWriter;
+
+ private transient CompactService compactService;
+
+ // collecting requests during the current checkpoint
+ private List<CompactorRequest> collectingRequests = new ArrayList<>();
+ // checkpoint id to its collecting requests, the requests is added when checkpointing, and are
+ // moved to compacting requests once submitted.
+ private final TreeMap<Long, List<CompactorRequest>> checkpointRequests = new TreeMap<>();
+ // the currently compacting requests and their completable futures, completed requests are
+ // removed and the results are emitted at #prepareSnapshotPreBarrier
+ private final List<Tuple2<CompactorRequest, CompletableFuture<Iterable<FileSinkCommittable>>>>
+ compactingRequests = new LinkedList<>();
+
+ // state combining checkpointRequests and compactingRequests, compactingRequests will be
+ // submitted again while restoring
+ private ListState<Map<Long, List<CompactorRequest>>> remainingRequestsState;
+
+ public CompactorOperator(
+ FileCompactStrategy strategy,
+ SimpleVersionedSerializer<FileSinkCommittable> committableSerializer,
+ FileCompactor fileCompactor,
+ BucketWriter<?, String> bucketWriter) {
+ this.strategy = strategy;
+ this.committableSerializer = committableSerializer;
+ this.fileCompactor = fileCompactor;
+ this.bucketWriter = bucketWriter;
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+ this.compactService =
+ new CompactService(strategy.getNumCompactThreads(), fileCompactor, bucketWriter);
+ compactService.open();
+
+ // submit all requests that is already submitted before restored checkpoint
+ submitUntil(SUBMITTED_ID);
+ }
+
+ @Override
+ public void processElement(StreamRecord<CompactorRequest> element) throws Exception {
+ collectingRequests.add(element.getValue());
+ }
+
+ @Override
+ public void endInput() throws Exception {
+ // add collecting requests into the final snapshot
+ checkpointRequests.put(Long.MAX_VALUE, collectingRequests);
+ collectingRequests = new ArrayList<>();
+
+ // submit all requests and wait until they are done
+ submitUntil(Long.MAX_VALUE);
+ assert checkpointRequests.isEmpty();
+
+ getAllTasksFuture().join();
+ emitCompacted(null);
+ assert compactingRequests.isEmpty();
+ }
+
+ @Override
+ public void notifyCheckpointComplete(long checkpointId) throws Exception {
+ super.notifyCheckpointComplete(checkpointId);
+ submitUntil(checkpointId);
+ }
+
+ @Override
+ public void prepareSnapshotPreBarrier(long checkpointId) throws Exception {
+ emitCompacted(checkpointId);
+ }
+
+ @Override
+ public void snapshotState(StateSnapshotContext context) throws Exception {
+ super.snapshotState(context);
+
+ // add collecting requests during the checkpoint into the snapshot
+ checkpointRequests.put(context.getCheckpointId(), collectingRequests);
+ collectingRequests = new ArrayList<>();
+
+ // snapshot all compacting requests as well, including the requests that are not finished
+ // when invoking prepareSnapshotPreBarrier but finished now, since they are not emitted yet
+ Map<Long, List<CompactorRequest>> requests = new HashMap<>(checkpointRequests);
+ requests.computeIfAbsent(SUBMITTED_ID, id -> new ArrayList<>())
+ .addAll(compactingRequests.stream().map(r -> r.f0).collect(Collectors.toList()));
+ remainingRequestsState.update(Collections.singletonList(requests));
+ }
+
+ @Override
+ public void initializeState(StateInitializationContext context) throws Exception {
+ super.initializeState(context);
+
+ remainingRequestsState =
+ new SimpleVersionedListState<>(
+ context.getOperatorStateStore()
+ .getListState(REMAINING_REQUESTS_RAW_STATES_DESC),
+ new RemainingRequestsSerializer(
+ new CompactorRequestSerializer(committableSerializer)));
+
+ Iterable<Map<Long, List<CompactorRequest>>> stateRemaining = remainingRequestsState.get();
+ if (stateRemaining != null) {
+ for (Map<Long, List<CompactorRequest>> requests : stateRemaining) {
+ // elements can be more than one when redistributed after parallelism changing
+ for (Map.Entry<Long, List<CompactorRequest>> e : requests.entrySet()) {
+ List<CompactorRequest> list =
+ checkpointRequests.computeIfAbsent(e.getKey(), id -> new ArrayList<>());
+ list.addAll(e.getValue());
+ }
+ }
+ }
+ // open is called afterwards, so do not submit here since compact service is not opened yet
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (compactService != null) {
+ compactService.close();
+ }
+ }
+
+ private void submitUntil(long checkpointId) {
+ NavigableMap<Long, List<CompactorRequest>> canSubmit =
+ checkpointRequests.subMap(Long.MIN_VALUE, true, checkpointId, true);
+ for (Entry<Long, List<CompactorRequest>> requestEntry : canSubmit.entrySet()) {
+ for (CompactorRequest request : requestEntry.getValue()) {
+ CompletableFuture<Iterable<FileSinkCommittable>> resultFuture =
+ new CompletableFuture<>();
+ compactingRequests.add(new Tuple2<>(request, resultFuture));
+ compactService.submit(request, resultFuture);
+ }
+ }
+ canSubmit.clear();
+ }
+
+ private void emitCompacted(@Nullable Long checkpointId) throws Exception {
+ List<FileSinkCommittable> compacted = new ArrayList<>();
+ Iterator<Tuple2<CompactorRequest, CompletableFuture<Iterable<FileSinkCommittable>>>> iter =
+ compactingRequests.iterator();
+ while (iter.hasNext()) {
+ Tuple2<CompactorRequest, CompletableFuture<Iterable<FileSinkCommittable>>> compacting =
+ iter.next();
+ CompletableFuture<Iterable<FileSinkCommittable>> future = compacting.f1;
+ if (future.isDone()) {
+ iter.remove();
+ // Exception is thrown if it's completed exceptionally
+ for (FileSinkCommittable c : future.get()) {
+ compacted.add(c);
+ }
+ }
+ }
+
+ if (compacted.isEmpty()) {
+ return;
+ }
+
+ // A summary must be sent before all results during this checkpoint
+ CommittableSummary<FileSinkCommittable> summary =
+ new CommittableSummary<>(
+ getRuntimeContext().getIndexOfThisSubtask(),
+ getRuntimeContext().getNumberOfParallelSubtasks(),
+ checkpointId,
+ compacted.size(),
+ compacted.size(),
+ 0);
+ output.collect(new StreamRecord<>(summary));
+ for (FileSinkCommittable c : compacted) {
+ CommittableWithLineage<FileSinkCommittable> comm =
+ new CommittableWithLineage<>(
+ c, checkpointId, getRuntimeContext().getIndexOfThisSubtask());
+ output.collect(new StreamRecord<>(comm));
+ }
+ }
+
+ @VisibleForTesting
+ public CompletableFuture<?> getAllTasksFuture() {
+ return CompletableFuture.allOf(
+ compactingRequests.stream().map(r -> r.f1).toArray(CompletableFuture[]::new));
+ }
+
+ static class RemainingRequestsSerializer
+ implements SimpleVersionedSerializer<Map<Long, List<CompactorRequest>>> {
+
+ private static final int MAGIC_NUMBER = 0xa946be83;
+
+ private final CompactorRequestSerializer requestSerializer;
+
+ RemainingRequestsSerializer(CompactorRequestSerializer requestSerializer) {
+ this.requestSerializer = requestSerializer;
+ }
+
+ @Override
+ public int getVersion() {
+ return 1;
+ }
+
+ @Override
+ public byte[] serialize(Map<Long, List<CompactorRequest>> remainingRequests)
+ throws IOException {
+ DataOutputSerializer out = new DataOutputSerializer(256);
+ out.writeInt(MAGIC_NUMBER);
+ serializeV1(remainingRequests, out);
+ return out.getCopyOfBuffer();
+ }
+
+ @Override
+ public Map<Long, List<CompactorRequest>> deserialize(int version, byte[] serialized)
+ throws IOException {
+ DataInputDeserializer in = new DataInputDeserializer(serialized);
+
+ switch (version) {
+ case 1:
+ validateMagicNumber(in);
+ return deserializeV1(in);
+ default:
+ throw new IOException("Unrecognized version or corrupt state: " + version);
+ }
+ }
+
+ private void serializeV1(
+ Map<Long, List<CompactorRequest>> request, DataOutputSerializer out)
+ throws IOException {
+ out.writeInt(request.size());
+ for (Map.Entry<Long, List<CompactorRequest>> e : request.entrySet()) {
+ out.writeLong(e.getKey());
+ SimpleVersionedSerialization.writeVersionAndSerializeList(
+ requestSerializer, e.getValue(), out);
+ }
+ }
+
+ private Map<Long, List<CompactorRequest>> deserializeV1(DataInputDeserializer in)
+ throws IOException {
+ int size = in.readInt();
+ Map<Long, List<CompactorRequest>> requestMap = new HashMap<>(size);
+ for (int i = 0; i < size; i++) {
+ long cpId = in.readLong();
+ List<CompactorRequest> requests =
+ SimpleVersionedSerialization.readVersionAndDeserializeList(
+ requestSerializer, in);
+ requestMap.put(cpId, requests);
+ }
+ return requestMap;
+ }
+
+ private static void validateMagicNumber(DataInputView in) throws IOException {
+ int magicNumber = in.readInt();
+ if (magicNumber != MAGIC_NUMBER) {
+ throw new IOException(
+ String.format("Corrupt data: Unexpected magic number %08X", magicNumber));
+ }
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorFactory.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorFactory.java
new file mode 100644
index 0000000..edbac27
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorFactory.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.compactor.FileCompactStrategy;
+import org.apache.flink.connector.file.sink.compactor.FileCompactor;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+import org.apache.flink.util.function.SerializableSupplierWithException;
+
+import java.io.IOException;
+
+/** Factory for {@link CompactorOperator}. */
+@Internal
+public class CompactorOperatorFactory
+ extends AbstractStreamOperatorFactory<CommittableMessage<FileSinkCommittable>>
+ implements OneInputStreamOperatorFactory<
+ CompactorRequest, CommittableMessage<FileSinkCommittable>> {
+
+ private final FileCompactStrategy strategy;
+ private final FileCompactor fileCompactor;
+ private final SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier;
+ private final SerializableSupplierWithException<BucketWriter<?, String>, IOException>
+ bucketWriterProvider;
+
+ public CompactorOperatorFactory(
+ FileCompactStrategy strategy,
+ FileCompactor fileCompactor,
+ SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier,
+ SerializableSupplierWithException<BucketWriter<?, String>, IOException>
+ bucketWriterProvider) {
+ this.strategy = strategy;
+ this.fileCompactor = fileCompactor;
+ this.committableSerializerSupplier = committableSerializerSupplier;
+ this.bucketWriterProvider = bucketWriterProvider;
+ }
+
+ @Override
+ public <T extends StreamOperator<CommittableMessage<FileSinkCommittable>>>
+ T createStreamOperator(
+ StreamOperatorParameters<CommittableMessage<FileSinkCommittable>> parameters) {
+ try {
+ final CompactorOperator compactOperator =
+ new CompactorOperator(
+ strategy,
+ committableSerializerSupplier.get(),
+ fileCompactor,
+ bucketWriterProvider.get());
+ compactOperator.setup(
+ parameters.getContainingTask(),
+ parameters.getStreamConfig(),
+ parameters.getOutput());
+ return (T) compactOperator;
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Cannot create commit operator for "
+ + parameters.getStreamConfig().getOperatorName(),
+ e);
+ }
+ }
+
+ @Override
+ public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {
+ return CompactorOperator.class;
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorStateHandler.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorStateHandler.java
new file mode 100644
index 0000000..70e6458
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorStateHandler.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.state.CheckpointListener;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.compactor.FileCompactor;
+import org.apache.flink.connector.file.sink.compactor.IdenticalFileCompactor;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorOperator.RemainingRequestsSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableSummary;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.BoundedOneInput;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.util.SimpleVersionedListState;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.types.Either;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.flink.connector.file.sink.compactor.operator.CompactorOperator.REMAINING_REQUESTS_RAW_STATES_DESC;
+import static org.apache.flink.util.Preconditions.checkState;
+
+/** Handler that processes the state of {@link CompactorOperator} when compaction is disabled. */
+@Internal
+public class CompactorOperatorStateHandler
+ extends AbstractStreamOperator<CommittableMessage<FileSinkCommittable>>
+ implements OneInputStreamOperator<
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>,
+ CommittableMessage<FileSinkCommittable>>,
+ BoundedOneInput,
+ CheckpointListener {
+
+ private final SimpleVersionedSerializer<FileSinkCommittable> committableSerializer;
+ private final BucketWriter<?, String> bucketWriter;
+
+ private final FileCompactor fileCompactor;
+
+ private transient CompactService compactService;
+
+ // Flag indicating the in-progress file of the previous run from the writer has been received
+ // and processed.
+ private boolean writerStateDrained = false;
+
+ // Flag indicating all compaction related states are drained, the operator can now pass through
+ // everything.
+ private boolean stateDrained = false;
+
+ // There may be a in-progress file of the previous run that we have to process as a compaction
+ // request first, or the file is invisible after committing.
+ // We have to hold the summary and committables (of this run), and send them along with the
+ // result of this compaction request, as well as the results of the remaining requests of this
+ // operator, if there are.
+ private CommittableSummary<FileSinkCommittable> holdingSummary;
+ private List<CommittableMessage<FileSinkCommittable>> holdingMessages;
+ private final List<CommittableMessage<FileSinkCommittable>> compactingMessages =
+ new ArrayList<>();
+
+ private final List<Tuple2<CompactorRequest, CompletableFuture<Iterable<FileSinkCommittable>>>>
+ compactingRequests = new ArrayList<>();
+ private SimpleVersionedListState<Map<Long, List<CompactorRequest>>> remainingRequestsState;
+ private Iterable<Map<Long, List<CompactorRequest>>> stateRemaining;
+
+ public CompactorOperatorStateHandler(
+ SimpleVersionedSerializer<FileSinkCommittable> committableSerializer,
+ BucketWriter<?, String> bucketWriter) {
+ this.committableSerializer = committableSerializer;
+ this.bucketWriter = bucketWriter;
+
+ this.fileCompactor = new IdenticalFileCompactor();
+ }
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+ this.compactService = new CompactService(1, fileCompactor, bucketWriter);
+ compactService.open();
+
+ if (stateRemaining != null) {
+ for (Map<Long, List<CompactorRequest>> requests : stateRemaining) {
+ for (Map.Entry<Long, List<CompactorRequest>> e : requests.entrySet()) {
+ for (CompactorRequest request : e.getValue()) {
+ // each toCompact must be wrapped with a single compact request, since
+ // multiple files compacting is not available now
+ List<FileSinkCommittable> toCompactList = request.getCommittableToCompact();
+ List<FileSinkCommittable> toPassThrough =
+ request.getCommittableToPassthrough();
+
+ String bucketId =
+ !toCompactList.isEmpty()
+ ? toCompactList.get(0).getBucketId()
+ : toPassThrough.get(0).getBucketId();
+
+ for (FileSinkCommittable toCompact : toCompactList) {
+ CompactorRequest compactRequest = new CompactorRequest(bucketId);
+ compactRequest.addToCompact(toCompact);
+ submit(compactRequest);
+ }
+
+ CompactorRequest passThroughRequest = new CompactorRequest(bucketId);
+ toPassThrough.forEach(passThroughRequest::addToPassthrough);
+ submit(passThroughRequest);
+ }
+ }
+ }
+ }
+ stateRemaining = null;
+ }
+
+ @Override
+ public void processElement(
+ StreamRecord<Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>> element)
+ throws Exception {
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest> record =
+ element.getValue();
+ if (stateDrained) {
+ // all input should be committable messages to pass through
+ output.collect(new StreamRecord<>(record.left()));
+ return;
+ }
+
+ if (record.isRight()) {
+ submit(element.getValue().right());
+ return;
+ }
+
+ CommittableMessage<FileSinkCommittable> message = record.left();
+ if (message instanceof CommittableSummary) {
+ checkState(holdingSummary == null, "Duplicate summary before the first checkpoint.");
+ holdingSummary = (CommittableSummary<FileSinkCommittable>) message;
+ holdingMessages = new ArrayList<>(holdingSummary.getNumberOfCommittables());
+ } else {
+ boolean compacting = false;
+ CommittableWithLineage<FileSinkCommittable> committableWithLineage =
+ (CommittableWithLineage<FileSinkCommittable>) message;
+ if (committableWithLineage.getCommittable().hasPendingFile()) {
+ FileSinkCommittable committable = committableWithLineage.getCommittable();
+ PendingFileRecoverable pendingFile = committable.getPendingFile();
+ if (pendingFile.getPath() != null
+ && pendingFile.getPath().getName().startsWith(".")) {
+ // The pending file is the in-progress file of the previous run, which
+ // should be committed and compacted before sending to the committer.
+ CompactorRequest request = new CompactorRequest(committable.getBucketId());
+ request.addToCompact(committable);
+ submit(request);
+
+ compacting = true;
+ compactingMessages.add(message);
+ } else {
+ // A normal file is received, indicating the writer state is drained.
+ writerStateDrained = true;
+ if (compactingMessages.isEmpty() && compactingRequests.isEmpty()) {
+ // No state needs to be handled, the holding summary and all committable
+ // messages can be sent eagerly
+ checkState(holdingSummary != null);
+ output.collect(new StreamRecord<>(holdingSummary));
+ holdingSummary = null;
+
+ this.stateDrained = true;
+ output.collect(new StreamRecord<>(committableWithLineage));
+ }
+ }
+ }
+ if (!compacting && !stateDrained) {
+ // Compacting messages should not be added
+ // If the state is drained, no further messages need to be added
+ holdingMessages.add(message);
+ }
+ }
+ }
+
+ @Override
+ public void prepareSnapshotPreBarrier(long checkpointId) throws Exception {
+ super.prepareSnapshotPreBarrier(checkpointId);
+ if (stateDrained) {
+ return;
+ }
+ drain();
+ // The operator is stateless once drain is called. snapshotState is not necessary.
+ }
+
+ @Override
+ public void endInput() throws Exception {
+ if (!stateDrained) {
+ drain();
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ if (compactService != null) {
+ compactService.close();
+ }
+ }
+
+ private void submit(CompactorRequest request) {
+ CompletableFuture<Iterable<FileSinkCommittable>> resultFuture = new CompletableFuture<>();
+ compactService.submit(request, resultFuture);
+ compactingRequests.add(new Tuple2<>(request, resultFuture));
+ }
+
+ private void drain() throws ExecutionException, InterruptedException {
+ checkState(holdingSummary != null);
+ checkState(
+ holdingSummary.getNumberOfPendingCommittables()
+ == holdingSummary.getNumberOfCommittables()
+ && holdingSummary.getNumberOfCommittables()
+ == holdingMessages.size() + compactingMessages.size());
+
+ Long checkpointId =
+ holdingSummary.getCheckpointId().isPresent()
+ ? holdingSummary.getCheckpointId().getAsLong()
+ : null;
+ int subtaskId = holdingSummary.getSubtaskId();
+
+ if (!compactingRequests.isEmpty()) {
+ CompletableFuture.allOf(
+ compactingRequests.stream()
+ .map(r -> r.f1)
+ .toArray(CompletableFuture[]::new))
+ .join();
+
+ for (Tuple2<CompactorRequest, CompletableFuture<Iterable<FileSinkCommittable>>>
+ compacting : compactingRequests) {
+ CompletableFuture<Iterable<FileSinkCommittable>> future = compacting.f1;
+ checkState(future.isDone());
+ // Exception is thrown if it's completed exceptionally
+ for (FileSinkCommittable c : future.get()) {
+ holdingMessages.add(new CommittableWithLineage<>(c, checkpointId, subtaskId));
+ }
+ }
+ }
+
+ // Appending the compacted committable to the holding summary
+ CommittableSummary<FileSinkCommittable> summary =
+ new CommittableSummary<>(
+ holdingSummary.getSubtaskId(),
+ holdingSummary.getNumberOfSubtasks(),
+ holdingSummary.getCheckpointId().isPresent()
+ ? holdingSummary.getCheckpointId().getAsLong()
+ : null,
+ holdingMessages.size(),
+ holdingMessages.size(),
+ holdingSummary.getNumberOfFailedCommittables());
+ output.collect(new StreamRecord<>(summary));
+ for (CommittableMessage<FileSinkCommittable> committable : holdingMessages) {
+ output.collect(new StreamRecord<>(committable));
+ }
+
+ // Remaining requests should be all done and their results are all emitted.
+ // From now on the operator is stateless.
+ remainingRequestsState.clear();
+
+ compactingRequests.clear();
+ compactingMessages.clear();
+ holdingSummary = null;
+ holdingMessages = null;
+
+ if (writerStateDrained) {
+ // We can pass through everything if the writer state is also drained.
+ stateDrained = true;
+ compactService.close();
+ compactService = null;
+ }
+ }
+
+ @Override
+ public void initializeState(StateInitializationContext context) throws Exception {
+ super.initializeState(context);
+
+ remainingRequestsState =
+ new SimpleVersionedListState<>(
+ context.getOperatorStateStore()
+ .getListState(REMAINING_REQUESTS_RAW_STATES_DESC),
+ new RemainingRequestsSerializer(
+ new CompactorRequestSerializer(committableSerializer)));
+
+ stateRemaining = remainingRequestsState.get();
+
+ // stateDrained can not be determined here, since even if the stateRemaining is empty,
+ // there may still be some requests from the coordinator and a in-progress file in the file
+ // writer
+ }
+
+ @VisibleForTesting
+ public boolean isWriterStateDrained() {
+ return writerStateDrained;
+ }
+
+ @VisibleForTesting
+ public boolean isStateDrained() {
+ return stateDrained;
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorStateHandlerFactory.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorStateHandlerFactory.java
new file mode 100644
index 0000000..2d36fee
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorOperatorStateHandlerFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+import org.apache.flink.types.Either;
+import org.apache.flink.util.function.SerializableSupplierWithException;
+
+import java.io.IOException;
+
+/** Factory for {@link CompactorOperatorStateHandler}. */
+@Internal
+public class CompactorOperatorStateHandlerFactory
+ extends AbstractStreamOperatorFactory<CommittableMessage<FileSinkCommittable>>
+ implements OneInputStreamOperatorFactory<
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>,
+ CommittableMessage<FileSinkCommittable>> {
+
+ private final SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier;
+ private final SerializableSupplierWithException<BucketWriter<?, String>, IOException>
+ bucketWriterProvider;
+
+ public CompactorOperatorStateHandlerFactory(
+ SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier,
+ SerializableSupplierWithException<BucketWriter<?, String>, IOException>
+ bucketWriterProvider) {
+ this.committableSerializerSupplier = committableSerializerSupplier;
+ this.bucketWriterProvider = bucketWriterProvider;
+ }
+
+ @Override
+ public <T extends StreamOperator<CommittableMessage<FileSinkCommittable>>>
+ T createStreamOperator(
+ StreamOperatorParameters<CommittableMessage<FileSinkCommittable>> parameters) {
+ try {
+ final CompactorOperatorStateHandler handler =
+ new CompactorOperatorStateHandler(
+ committableSerializerSupplier.get(), bucketWriterProvider.get());
+ handler.setup(
+ parameters.getContainingTask(),
+ parameters.getStreamConfig(),
+ parameters.getOutput());
+ return (T) handler;
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Cannot create commit operator for "
+ + parameters.getStreamConfig().getOperatorName(),
+ e);
+ }
+ }
+
+ @Override
+ public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {
+ return CompactorOperator.class;
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequest.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequest.java
new file mode 100644
index 0000000..86bc78f
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.file.sink.FileSink;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Request of file compacting for {@link FileSink}. */
+@Internal
+public class CompactorRequest implements Serializable {
+ private final String bucketId;
+ private final List<FileSinkCommittable> committableToCompact;
+ private final List<FileSinkCommittable> committableToPassthrough;
+
+ public CompactorRequest(String bucketId) {
+ this.bucketId = bucketId;
+ this.committableToCompact = new ArrayList<>();
+ this.committableToPassthrough = new ArrayList<>();
+ }
+
+ public CompactorRequest(
+ String bucketId,
+ List<FileSinkCommittable> committableToCompact,
+ List<FileSinkCommittable> committableToPassthrough) {
+ this.bucketId = bucketId;
+ this.committableToCompact = committableToCompact;
+ this.committableToPassthrough = committableToPassthrough;
+ }
+
+ public void addToCompact(FileSinkCommittable committable) {
+ committableToCompact.add(committable);
+ }
+
+ public void addToPassthrough(FileSinkCommittable committable) {
+ committableToPassthrough.add(committable);
+ }
+
+ public String getBucketId() {
+ return bucketId;
+ }
+
+ public List<FileSinkCommittable> getCommittableToCompact() {
+ return committableToCompact;
+ }
+
+ public List<FileSinkCommittable> getCommittableToPassthrough() {
+ return committableToPassthrough;
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestSerializer.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestSerializer.java
new file mode 100644
index 0000000..e00e32f
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestSerializer.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.core.io.SimpleVersionedSerialization;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputSerializer;
+
+import java.io.IOException;
+import java.util.List;
+
+/** Versioned serializer for {@link CompactorRequest}. */
+@Internal
+public class CompactorRequestSerializer implements SimpleVersionedSerializer<CompactorRequest> {
+
+ private final SimpleVersionedSerializer<FileSinkCommittable> committableSerializer;
+
+ private static final int MAGIC_NUMBER = 0x2fc61e19;
+
+ public CompactorRequestSerializer(
+ SimpleVersionedSerializer<FileSinkCommittable> committableSerializer) {
+ this.committableSerializer = committableSerializer;
+ }
+
+ @Override
+ public int getVersion() {
+ return 1;
+ }
+
+ @Override
+ public byte[] serialize(CompactorRequest request) throws IOException {
+ DataOutputSerializer out = new DataOutputSerializer(256);
+ out.writeInt(MAGIC_NUMBER);
+ serializeV1(request, out);
+ return out.getCopyOfBuffer();
+ }
+
+ @Override
+ public CompactorRequest deserialize(int version, byte[] serialized) throws IOException {
+ DataInputDeserializer in = new DataInputDeserializer(serialized);
+
+ switch (version) {
+ case 1:
+ validateMagicNumber(in);
+ return deserializeV1(in);
+ default:
+ throw new IOException("Unrecognized version or corrupt state: " + version);
+ }
+ }
+
+ private void serializeV1(CompactorRequest request, DataOutputSerializer out)
+ throws IOException {
+ out.writeUTF(request.getBucketId());
+ SimpleVersionedSerialization.writeVersionAndSerializeList(
+ committableSerializer, request.getCommittableToCompact(), out);
+ SimpleVersionedSerialization.writeVersionAndSerializeList(
+ committableSerializer, request.getCommittableToPassthrough(), out);
+ }
+
+ private CompactorRequest deserializeV1(DataInputDeserializer in) throws IOException {
+ String bucketId = in.readUTF();
+ List<FileSinkCommittable> committableToCompact =
+ SimpleVersionedSerialization.readVersionAndDeserializeList(
+ committableSerializer, in);
+ List<FileSinkCommittable> committableToPassthrough =
+ SimpleVersionedSerialization.readVersionAndDeserializeList(
+ committableSerializer, in);
+ return new CompactorRequest(bucketId, committableToCompact, committableToPassthrough);
+ }
+
+ private static void validateMagicNumber(DataInputView in) throws IOException {
+ int magicNumber = in.readInt();
+ if (magicNumber != MAGIC_NUMBER) {
+ throw new IOException(
+ String.format("Corrupt data: Unexpected magic number %08X", magicNumber));
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestTypeInfo.java b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestTypeInfo.java
new file mode 100644
index 0000000..b9bbf87
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/main/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestTypeInfo.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializerTypeSerializerProxy;
+import org.apache.flink.util.function.SerializableSupplierWithException;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/** Type information of {@link CompactorRequest}. Unsuitable for state. */
+@Internal
+public class CompactorRequestTypeInfo extends TypeInformation<CompactorRequest> {
+
+ private final SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier;
+
+ public CompactorRequestTypeInfo(
+ SerializableSupplierWithException<
+ SimpleVersionedSerializer<FileSinkCommittable>, IOException>
+ committableSerializerSupplier) {
+ this.committableSerializerSupplier = committableSerializerSupplier;
+ }
+
+ @Override
+ public boolean isBasicType() {
+ return false;
+ }
+
+ @Override
+ public boolean isTupleType() {
+ return false;
+ }
+
+ @Override
+ public int getArity() {
+ return 1;
+ }
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
+
+ @Override
+ public Class<CompactorRequest> getTypeClass() {
+ return CompactorRequest.class;
+ }
+
+ @Override
+ public boolean isKeyType() {
+ return false;
+ }
+
+ @Override
+ public TypeSerializer<CompactorRequest> createSerializer(ExecutionConfig config) {
+ return new SimpleVersionedSerializerTypeSerializerProxy<>(
+ () -> new CompactorRequestSerializer(createCommittableSerializer()));
+ }
+
+ @Override
+ public String toString() {
+ return "CompactorRequestTypeInfo{" + "serializer=" + createCommittableSerializer() + '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || !canEqual(o)) {
+ return false;
+ }
+ CompactorRequestTypeInfo that = (CompactorRequestTypeInfo) o;
+ return Objects.equals(
+ createCommittableSerializer().getClass(),
+ that.createCommittableSerializer().getClass());
+ }
+
+ @Override
+ public boolean canEqual(Object obj) {
+ return obj instanceof CompactorRequestTypeInfo;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(createCommittableSerializer().getClass());
+ }
+
+ private SimpleVersionedSerializer<FileSinkCommittable> createCommittableSerializer() {
+ try {
+ return committableSerializerSupplier.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCommittableSerializerMigrationTest.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCommittableSerializerMigrationTest.java
index 87968821..0d5c96e 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCommittableSerializerMigrationTest.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/FileSinkCommittableSerializerMigrationTest.java
@@ -89,7 +89,7 @@ public class FileSinkCommittableSerializerMigrationTest {
OutputStreamBasedInProgressFileRecoverable recoverable =
new OutputStreamBasedInProgressFileRecoverable(resumeRecoverable);
- FileSinkCommittable committable = new FileSinkCommittable(recoverable);
+ FileSinkCommittable committable = new FileSinkCommittable("0", recoverable);
byte[] bytes = serializer.serialize(committable);
Files.write(path.resolve("committable"), bytes);
@@ -134,7 +134,7 @@ public class FileSinkCommittableSerializerMigrationTest {
OutputStreamBasedPendingFileRecoverable recoverable =
new OutputStreamBasedPendingFileRecoverable(commitRecoverable);
- FileSinkCommittable committable = new FileSinkCommittable(recoverable);
+ FileSinkCommittable committable = new FileSinkCommittable("0", recoverable);
byte[] bytes = serializer.serialize(committable);
Files.write(path.resolve("committable"), bytes);
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/AbstractCompactTestBase.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/AbstractCompactTestBase.java
new file mode 100644
index 0000000..cffada6
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/AbstractCompactTestBase.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.core.fs.Path;
+
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/** Test base for compact operators. */
+public abstract class AbstractCompactTestBase {
+
+ @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder();
+
+ public static final int TARGET_SIZE = 9;
+
+ Path folder;
+
+ @Before
+ public void before() throws IOException {
+ folder = new Path(TEMP_FOLDER.newFolder().getPath());
+ }
+
+ Path newFile(String name, int len) throws IOException {
+ Path path = new Path(folder, name);
+ File file = new File(path.getPath());
+ file.delete();
+ file.createNewFile();
+
+ try (FileOutputStream out = new FileOutputStream(file)) {
+ for (int i = 0; i < len; i++) {
+ out.write(i);
+ }
+ }
+ return path;
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/CompactCoordinatorTest.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/CompactCoordinatorTest.java
new file mode 100644
index 0000000..0fbfb9b
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/CompactCoordinatorTest.java
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.api.java.typeutils.runtime.EitherSerializer;
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.FileSinkCommittableSerializer;
+import org.apache.flink.connector.file.sink.compactor.FileCompactStrategy.Builder;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactCoordinator;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactCoordinatorStateHandler;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorRequest;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorRequestSerializer;
+import org.apache.flink.connector.file.sink.utils.FileSinkTestUtils;
+import org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable;
+import org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializerTypeSerializerProxy;
+import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessageSerializer;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.types.Either;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+/** Test for {@link CompactCoordinator}. */
+public class CompactCoordinatorTest extends AbstractCompactTestBase {
+
+ @Test
+ public void testSizeThreshold() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(10).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ FileSinkCommittable committable0 = committable("0", ".0", 5);
+ FileSinkCommittable committable1 = committable("0", ".1", 6);
+ harness.processElement(message(committable0));
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.processElement(message(committable1));
+
+ List<CompactorRequest> results = harness.extractOutputValues();
+ Assert.assertEquals(1, results.size());
+ assertToCompact(results.get(0), committable0, committable1);
+
+ harness.processElement(message(committable("0", ".2", 5)));
+ harness.processElement(message(committable("1", ".0", 5)));
+
+ Assert.assertEquals(1, harness.extractOutputValues().size());
+ }
+ }
+
+ @Test
+ public void testCompactOnCheckpoint() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().enableCompactionOnCheckpoint(1).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ FileSinkCommittable committable0 = committable("0", ".0", 5);
+ FileSinkCommittable committable1 = committable("0", ".1", 6);
+ FileSinkCommittable committable2 = committable("0", ".2", 5);
+ FileSinkCommittable committable3 = committable("1", ".0", 5);
+
+ harness.processElement(message(committable0));
+ harness.processElement(message(committable1));
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(1);
+ harness.snapshot(1, 1);
+
+ Assert.assertEquals(1, harness.extractOutputValues().size());
+
+ harness.processElement(message(committable2));
+ harness.processElement(message(committable3));
+
+ Assert.assertEquals(1, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(2);
+ harness.snapshot(2, 2);
+
+ List<CompactorRequest> results = harness.extractOutputValues();
+ Assert.assertEquals(3, results.size());
+ assertToCompact(results.get(0), committable0, committable1);
+ assertToCompact(results.get(1), committable2);
+ assertToCompact(results.get(2), committable3);
+ }
+ }
+
+ @Test
+ public void testCompactOverMultipleCheckpoints() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().enableCompactionOnCheckpoint(3).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ FileSinkCommittable committable0 = committable("0", ".0", 5);
+ FileSinkCommittable committable1 = committable("0", ".1", 6);
+
+ harness.processElement(message(committable0));
+ harness.processElement(message(committable1));
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(1);
+ harness.snapshot(1, 1);
+ harness.prepareSnapshotPreBarrier(2);
+ harness.snapshot(2, 2);
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(3);
+ harness.snapshot(3, 3);
+
+ List<CompactorRequest> results = harness.extractOutputValues();
+ Assert.assertEquals(1, results.size());
+ assertToCompact(results.get(0), committable0, committable1);
+ }
+ }
+
+ @Test
+ public void testCompactOnEndOfInput() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(10).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ FileSinkCommittable committable0 = committable("0", ".0", 5);
+
+ harness.processElement(message(committable0));
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(1);
+ harness.snapshot(1, 1);
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.endInput();
+
+ List<CompactorRequest> results = harness.extractOutputValues();
+ Assert.assertEquals(1, results.size());
+ assertToCompact(results.get(0), committable0);
+ }
+ }
+
+ @Test
+ public void testPassthrough() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(10).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ FileSinkCommittable cleanupToPassthrough = cleanupInprogress("0", ".0", 1);
+ FileSinkCommittable sizeUnavailableToPassthrough = committable("0", ".1", -1);
+ FileSinkCommittable pathNotHidToPassThrough = committable("0", "2", -1);
+ FileSinkCommittable normalCommittable = committable("0", ".3", 10);
+
+ harness.processElement(message(cleanupToPassthrough));
+ harness.processElement(message(sizeUnavailableToPassthrough));
+ harness.processElement(message(pathNotHidToPassThrough));
+ harness.processElement(message(normalCommittable));
+
+ List<CompactorRequest> results = harness.extractOutputValues();
+ Assert.assertEquals(1, results.size());
+ assertToCompact(results.get(0), normalCommittable);
+ assertToPassthrough(
+ results.get(0),
+ cleanupToPassthrough,
+ sizeUnavailableToPassthrough,
+ pathNotHidToPassThrough);
+ }
+ }
+
+ @Test
+ public void testRestore() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(10).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ FileSinkCommittable committable0 = committable("0", ".0", 5);
+ FileSinkCommittable committable1 = committable("0", ".1", 6);
+ FileSinkCommittable committable2 = committable("0", ".2", 5);
+ FileSinkCommittable committable3 = committable("1", ".0", 5);
+
+ OperatorSubtaskState state;
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ harness.processElement(message(committable0));
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(1);
+ state = harness.snapshot(1, 1);
+ }
+
+ coordinator = new CompactCoordinator(strategy, getTestCommittableSerializer());
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.initializeState(state);
+ harness.open();
+
+ harness.processElement(message(committable1));
+
+ Assert.assertEquals(1, harness.extractOutputValues().size());
+
+ harness.processElement(message(committable2));
+ harness.processElement(message(committable3));
+
+ Assert.assertEquals(1, harness.extractOutputValues().size());
+
+ harness.endInput();
+
+ List<CompactorRequest> results = harness.extractOutputValues();
+ Assert.assertEquals(3, results.size());
+ assertToCompact(results.get(0), committable0, committable1);
+ assertToCompact(results.get(1), committable2);
+ assertToCompact(results.get(2), committable3);
+ }
+ }
+
+ @Test
+ public void testRestoreWithChangedStrategy() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(100).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ FileSinkCommittable committable0 = committable("0", ".0", 5);
+ FileSinkCommittable committable1 = committable("0", ".1", 6);
+ FileSinkCommittable committable2 = committable("0", ".2", 7);
+ FileSinkCommittable committable3 = committable("0", ".3", 8);
+ FileSinkCommittable committable4 = committable("0", ".4", 9);
+ FileSinkCommittable committable5 = committable("0", ".5", 2);
+
+ OperatorSubtaskState state;
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ harness.processElement(message(committable0));
+ harness.processElement(message(committable1));
+ harness.processElement(message(committable2));
+ harness.processElement(message(committable3));
+ harness.processElement(message(committable4));
+
+ harness.prepareSnapshotPreBarrier(1);
+ state = harness.snapshot(1, 1);
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+ }
+
+ FileCompactStrategy changedStrategy = Builder.newBuilder().setSizeThreshold(10).build();
+ CompactCoordinator changedCoordinator =
+ new CompactCoordinator(changedStrategy, getTestCommittableSerializer());
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(changedCoordinator)) {
+ harness.setup();
+ harness.initializeState(state);
+ harness.open();
+
+ Assert.assertEquals(2, harness.extractOutputValues().size());
+
+ harness.processElement(message(committable5));
+
+ List<CompactorRequest> results = harness.extractOutputValues();
+ Assert.assertEquals(3, results.size());
+ assertToCompact(results.get(0), committable0, committable1);
+ assertToCompact(results.get(1), committable2, committable3);
+ assertToCompact(results.get(2), committable4, committable5);
+ }
+ }
+
+ @Test
+ public void testStateHandler() throws Exception {
+ FileCompactStrategy strategy = Builder.newBuilder().setSizeThreshold(10).build();
+ CompactCoordinator coordinator =
+ new CompactCoordinator(strategy, getTestCommittableSerializer());
+
+ // with . prefix
+ FileSinkCommittable committable0 = committable("0", ".0", 5);
+ FileSinkCommittable committable1 = committable("0", ".1", 6);
+
+ // without . prefix
+ FileSinkCommittable committable2 = committable("0", "2", 6);
+
+ OperatorSubtaskState state;
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>, CompactorRequest>
+ harness = new OneInputStreamOperatorTestHarness<>(coordinator)) {
+ harness.setup();
+ harness.open();
+
+ harness.processElement(message(committable0));
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(1);
+ state = harness.snapshot(1, 1);
+ }
+
+ CompactCoordinatorStateHandler handler =
+ new CompactCoordinatorStateHandler(getTestCommittableSerializer());
+ try (OneInputStreamOperatorTestHarness<
+ CommittableMessage<FileSinkCommittable>,
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>>
+ harness = new OneInputStreamOperatorTestHarness<>(handler)) {
+ harness.setup(
+ new EitherSerializer<>(
+ new SimpleVersionedSerializerTypeSerializerProxy<>(
+ () ->
+ new CommittableMessageSerializer<>(
+ getTestCommittableSerializer())),
+ new SimpleVersionedSerializerTypeSerializerProxy<>(
+ () ->
+ new CompactorRequestSerializer(
+ getTestCommittableSerializer()))));
+ harness.initializeState(state);
+ harness.open();
+
+ Assert.assertEquals(1, harness.extractOutputValues().size());
+
+ harness.processElement(message(committable1));
+ harness.processElement(message(committable2));
+
+ List<Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>> results =
+ harness.extractOutputValues();
+ Assert.assertEquals(3, results.size());
+
+ // restored request
+ Assert.assertTrue(results.get(0).isRight());
+ assertToCompact(results.get(0).right(), committable0);
+
+ // committable with . prefix should also be passed through
+ Assert.assertTrue(
+ results.get(1).isLeft()
+ && results.get(1).left() instanceof CommittableWithLineage);
+ Assert.assertEquals(
+ ((CommittableWithLineage<FileSinkCommittable>) results.get(1).left())
+ .getCommittable(),
+ committable1);
+
+ // committable without . prefix should be passed through normally
+ Assert.assertTrue(
+ results.get(2).isLeft()
+ && results.get(2).left() instanceof CommittableWithLineage);
+ Assert.assertEquals(
+ ((CommittableWithLineage<FileSinkCommittable>) results.get(2).left())
+ .getCommittable(),
+ committable2);
+ }
+ }
+
+ private StreamRecord<CommittableMessage<FileSinkCommittable>> message(
+ FileSinkCommittable committable) {
+ return new StreamRecord<>(new CommittableWithLineage<>(committable, 1L, 0), 0L);
+ }
+
+ private FileSinkCommittable committable(String bucketId, String name, int size)
+ throws IOException {
+ // put bucketId after name to keep the possible '.' prefix in name
+ return new FileSinkCommittable(
+ bucketId,
+ new TestPendingFileRecoverable(
+ newFile(name + "_" + bucketId, size <= 0 ? 1 : size), size));
+ }
+
+ private FileSinkCommittable cleanupInprogress(String bucketId, String name, int size)
+ throws IOException {
+ Path toCleanup = newFile(name + "_" + bucketId, size);
+ return new FileSinkCommittable(
+ bucketId, new TestInProgressFileRecoverable(toCleanup, size));
+ }
+
+ private SimpleVersionedSerializer<FileSinkCommittable> getTestCommittableSerializer() {
+ return new FileSinkCommittableSerializer(
+ new FileSinkTestUtils.SimpleVersionedWrapperSerializer<>(
+ FileSinkTestUtils.TestPendingFileRecoverable::new),
+ new FileSinkTestUtils.SimpleVersionedWrapperSerializer<>(
+ FileSinkTestUtils.TestInProgressFileRecoverable::new));
+ }
+
+ private void assertToCompact(CompactorRequest request, FileSinkCommittable... committables) {
+ List<FileSinkCommittable> committableToCompact = request.getCommittableToCompact();
+ Assert.assertArrayEquals(committables, committableToCompact.toArray());
+ }
+
+ private void assertToPassthrough(
+ CompactorRequest request, FileSinkCommittable... committables) {
+ List<FileSinkCommittable> committableToCompact = request.getCommittableToPassthrough();
+ Assert.assertArrayEquals(committables, committableToCompact.toArray());
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/CompactorOperatorTest.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/CompactorOperatorTest.java
new file mode 100644
index 0000000..6b7f397
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/CompactorOperatorTest.java
@@ -0,0 +1,534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor;
+
+import org.apache.flink.connector.file.sink.FileSinkCommittable;
+import org.apache.flink.connector.file.sink.FileSinkCommittableSerializer;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorOperator;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorOperatorStateHandler;
+import org.apache.flink.connector.file.sink.compactor.operator.CompactorRequest;
+import org.apache.flink.connector.file.sink.utils.FileSinkTestUtils;
+import org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestInProgressFileRecoverable;
+import org.apache.flink.connector.file.sink.utils.FileSinkTestUtils.TestPendingFileRecoverable;
+import org.apache.flink.connector.file.sink.utils.IntegerFileSinkTestDataUtils.IntDecoder;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
+import org.apache.flink.streaming.api.connector.sink2.CommittableMessage;
+import org.apache.flink.streaming.api.connector.sink2.CommittableSummary;
+import org.apache.flink.streaming.api.connector.sink2.CommittableWithLineage;
+import org.apache.flink.streaming.api.connector.sink2.SinkV2Assertions;
+import org.apache.flink.streaming.api.functions.sink.filesystem.BucketWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.CompactingFileWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.InProgressFileRecoverable;
+import org.apache.flink.streaming.api.functions.sink.filesystem.InProgressFileWriter.PendingFileRecoverable;
+import org.apache.flink.streaming.api.functions.sink.filesystem.OutputStreamBasedCompactingFileWriter;
+import org.apache.flink.streaming.api.functions.sink.filesystem.WriterProperties;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.types.Either;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/** Test for {@link CompactorOperator}. */
+public class CompactorOperatorTest extends AbstractCompactTestBase {
+
+ @Test
+ public void testCompact() throws Exception {
+ FileCompactor fileCompactor =
+ new RecordWiseFileCompactor<>(new DecoderBasedReader.Factory<>(IntDecoder::new));
+ CompactorOperator compactor = createTestOperator(fileCompactor);
+
+ try (OneInputStreamOperatorTestHarness<
+ CompactorRequest, CommittableMessage<FileSinkCommittable>>
+ harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
+ harness.setup();
+ harness.open();
+
+ harness.processElement(
+ request(
+ "0",
+ Arrays.asList(committable("0", ".0", 5), committable("0", ".1", 5)),
+ null));
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(1);
+ harness.snapshot(1, 1L);
+ harness.notifyOfCompletedCheckpoint(1);
+
+ compactor.getAllTasksFuture().join();
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(2);
+
+ // 1summary+1compacted+2cleanup
+ List<CommittableMessage<FileSinkCommittable>> results = harness.extractOutputValues();
+ Assert.assertEquals(4, results.size());
+ SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(0))
+ .hasPendingCommittables(3);
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(1))
+ .hasCommittable(committable("0", "compacted-0", 10));
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(2))
+ .hasCommittable(cleanupPath("0", ".0"));
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(3))
+ .hasCommittable(cleanupPath("0", ".1"));
+ }
+ }
+
+ @Test
+ public void testPassthrough() throws Exception {
+ FileCompactor fileCompactor =
+ new RecordWiseFileCompactor<>(new DecoderBasedReader.Factory<>(IntDecoder::new));
+ CompactorOperator compactor = createTestOperator(fileCompactor);
+
+ try (OneInputStreamOperatorTestHarness<
+ CompactorRequest, CommittableMessage<FileSinkCommittable>>
+ harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
+ harness.setup();
+ harness.open();
+
+ FileSinkCommittable cleanupInprogressRequest = cleanupInprogress("0", "0", 1);
+ FileSinkCommittable cleanupPathRequest = cleanupPath("0", "1");
+
+ harness.processElement(
+ request("0", null, Collections.singletonList(cleanupInprogressRequest)));
+ harness.processElement(
+ request("0", null, Collections.singletonList(cleanupPathRequest)));
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(1);
+ harness.snapshot(1, 1L);
+ harness.notifyOfCompletedCheckpoint(1);
+
+ compactor.getAllTasksFuture().join();
+
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ harness.prepareSnapshotPreBarrier(2);
+
+ List<CommittableMessage<FileSinkCommittable>> results = harness.extractOutputValues();
+ Assert.assertEquals(3, results.size());
+ SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(0))
+ .hasPendingCommittables(2);
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(1))
+ .hasCommittable(cleanupInprogressRequest);
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(2))
+ .hasCommittable(cleanupPathRequest);
+ }
+ }
+
+ @Test
+ public void testRestore() throws Exception {
+ FileCompactor fileCompactor =
+ new RecordWiseFileCompactor<>(new DecoderBasedReader.Factory<>(IntDecoder::new));
+ CompactorOperator compactor = createTestOperator(fileCompactor);
+
+ OperatorSubtaskState state;
+ try (OneInputStreamOperatorTestHarness<
+ CompactorRequest, CommittableMessage<FileSinkCommittable>>
+ harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
+ harness.setup();
+ harness.open();
+
+ harness.processElement(
+ request(
+ "0",
+ Arrays.asList(committable("0", ".0", 5), committable("0", ".1", 5)),
+ null));
+ harness.snapshot(1, 1L);
+
+ harness.processElement(
+ request(
+ "0",
+ Arrays.asList(committable("0", ".2", 5), committable("0", ".3", 5)),
+ null));
+
+ harness.notifyOfCompletedCheckpoint(1);
+
+ // request 1 is submitted and request 2 is pending
+ state = harness.snapshot(2, 2L);
+ }
+
+ compactor = createTestOperator(fileCompactor);
+ try (OneInputStreamOperatorTestHarness<
+ CompactorRequest, CommittableMessage<FileSinkCommittable>>
+ harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
+ harness.setup();
+ harness.initializeState(state);
+ harness.open();
+
+ // request 1 should be submitted
+ compactor.getAllTasksFuture().join();
+ harness.prepareSnapshotPreBarrier(3);
+
+ // the result of request 1 should be emitted
+ Assert.assertEquals(4, harness.extractOutputValues().size());
+
+ harness.snapshot(3, 3L);
+ harness.notifyOfCompletedCheckpoint(3L);
+
+ // request 2 should be submitted
+ compactor.getAllTasksFuture().join();
+ harness.prepareSnapshotPreBarrier(4);
+
+ // the result of request 2 should be emitted
+ Assert.assertEquals(8, harness.extractOutputValues().size());
+
+ // 1summary+1compacted+2cleanup * 2
+ List<CommittableMessage<FileSinkCommittable>> results = harness.extractOutputValues();
+ Assert.assertEquals(8, results.size());
+ SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(0))
+ .hasPendingCommittables(3);
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(1))
+ .hasCommittable(committable("0", "compacted-0", 10));
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(2))
+ .hasCommittable(cleanupPath("0", ".0"));
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(3))
+ .hasCommittable(cleanupPath("0", ".1"));
+
+ SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(4))
+ .hasPendingCommittables(3);
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(5))
+ .hasCommittable(committable("0", "compacted-2", 10));
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(6))
+ .hasCommittable(cleanupPath("0", ".2"));
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(7))
+ .hasCommittable(cleanupPath("0", ".3"));
+ }
+ }
+
+ @Test
+ public void testStateHandler() throws Exception {
+ FileCompactor fileCompactor =
+ new RecordWiseFileCompactor<>(new DecoderBasedReader.Factory<>(IntDecoder::new));
+ CompactorOperator compactor = createTestOperator(fileCompactor);
+
+ OperatorSubtaskState state;
+ try (OneInputStreamOperatorTestHarness<
+ CompactorRequest, CommittableMessage<FileSinkCommittable>>
+ harness = new OneInputStreamOperatorTestHarness<>(compactor)) {
+ harness.setup();
+ harness.open();
+
+ harness.processElement(
+ request(
+ "0",
+ Arrays.asList(committable("0", ".0", 1), committable("0", ".1", 2)),
+ null));
+ harness.snapshot(1, 1L);
+
+ harness.processElement(
+ request(
+ "0",
+ Arrays.asList(committable("0", ".2", 3), committable("0", ".3", 4)),
+ null));
+
+ harness.notifyOfCompletedCheckpoint(1);
+
+ // request 1 is submitted and request 2 is pending
+ state = harness.snapshot(2, 2L);
+ }
+
+ CompactorOperatorStateHandler handler =
+ new CompactorOperatorStateHandler(
+ getTestCommittableSerializer(), createTestBucketWriter());
+ try (OneInputStreamOperatorTestHarness<
+ Either<CommittableMessage<FileSinkCommittable>, CompactorRequest>,
+ CommittableMessage<FileSinkCommittable>>
+ harness = new OneInputStreamOperatorTestHarness<>(handler)) {
+ harness.setup();
+ harness.initializeState(state);
+ harness.open();
+
+ // remaining requests from coordinator
+ harness.processElement(
+ new StreamRecord<>(
+ Either.Right(
+ request(
+ "0",
+ Collections.singletonList(
+ committable("0", ".4", 5)),
+ null)
+ .getValue())));
+ harness.processElement(
+ new StreamRecord<>(
+ Either.Right(
+ request(
+ "0",
+ Collections.singletonList(
+ committable("0", ".5", 6)),
+ null)
+ .getValue())));
+
+ harness.processElement(
+ new StreamRecord<>(Either.Left(new CommittableSummary<>(0, 1, 3L, 2, 2, 0))));
+
+ // remaining in-progress file from file writer
+ harness.processElement(
+ new StreamRecord<>(
+ Either.Left(
+ new CommittableWithLineage<>(
+ committable("0", ".6", 7), 3L, 0))));
+
+ // new pending file written this time
+ harness.processElement(
+ new StreamRecord<>(
+ Either.Left(
+ new CommittableWithLineage<>(
+ committable("0", "7", 8), 3L, 0))));
+
+ Assert.assertTrue(handler.isWriterStateDrained());
+ Assert.assertFalse(handler.isStateDrained());
+
+ // the result should not be emitted yet, but all requests should already be submitted
+ Assert.assertEquals(0, harness.extractOutputValues().size());
+
+ compactor.getAllTasksFuture().join();
+ // state should be drained, and all results and holding messages should be emitted
+ harness.prepareSnapshotPreBarrier(3);
+
+ Assert.assertTrue(handler.isStateDrained());
+
+ // summary should be merged into one
+ // 1 summary+ 1 compacted + (1 compacted committable + 1 compacted cleanup) * 7
+ List<CommittableMessage<FileSinkCommittable>> results = harness.extractOutputValues();
+ Assert.assertEquals(16, results.size());
+ SinkV2Assertions.assertThat((CommittableSummary<?>) results.get(0))
+ .hasPendingCommittables(15);
+
+ List<FileSinkCommittable> expectedResult =
+ Arrays.asList(
+ committable("0", "7", 8),
+ committable("0", "compacted-0", 1),
+ cleanupPath("0", ".0"),
+ committable("0", "compacted-1", 2),
+ cleanupPath("0", ".1"),
+ committable("0", "compacted-2", 3),
+ cleanupPath("0", ".2"),
+ committable("0", "compacted-3", 4),
+ cleanupPath("0", ".3"),
+ committable("0", "compacted-4", 5),
+ cleanupPath("0", ".4"),
+ committable("0", "compacted-5", 6),
+ cleanupPath("0", ".5"),
+ committable("0", "compacted-6", 7),
+ cleanupPath("0", ".6"));
+
+ for (int i = 1; i < results.size(); ++i) {
+ SinkV2Assertions.assertThat((CommittableWithLineage<?>) results.get(i))
+ .hasCommittable(expectedResult.get(i - 1));
+ }
+ }
+ }
+
+ private StreamRecord<CompactorRequest> request(
+ String bucketId,
+ List<FileSinkCommittable> toCompact,
+ List<FileSinkCommittable> toPassthrough) {
+ return new StreamRecord<>(
+ new CompactorRequest(
+ bucketId,
+ toCompact == null ? new ArrayList<>() : toCompact,
+ toPassthrough == null ? new ArrayList<>() : toPassthrough),
+ 0L);
+ }
+
+ private FileSinkCommittable committable(String bucketId, String name, int size)
+ throws IOException {
+ // put bucketId after name to keep the possible '.' prefix in name
+ return new FileSinkCommittable(
+ bucketId,
+ new TestPendingFileRecoverable(
+ newFile(name + "_" + bucketId, size <= 0 ? 1 : size), size));
+ }
+
+ private FileSinkCommittable cleanupInprogress(String bucketId, String name, int size)
+ throws IOException {
+ Path toCleanup = newFile(name + "_" + bucketId, size);
+ return new FileSinkCommittable(
+ bucketId, new TestInProgressFileRecoverable(toCleanup, size));
+ }
+
+ private FileSinkCommittable cleanupPath(String bucketId, String name) throws IOException {
+ Path toCleanup = newFile(name + "_" + bucketId, 1);
+ return new FileSinkCommittable(bucketId, toCleanup);
+ }
+
+ private SimpleVersionedSerializer<FileSinkCommittable> getTestCommittableSerializer() {
+ return new FileSinkCommittableSerializer(
+ new FileSinkTestUtils.SimpleVersionedWrapperSerializer<>(
+ TestPendingFileRecoverable::new),
+ new FileSinkTestUtils.SimpleVersionedWrapperSerializer<>(
+ TestInProgressFileRecoverable::new));
+ }
+
+ private CompactorOperator createTestOperator(FileCompactor compactor) {
+ return new CompactorOperator(
+ FileCompactStrategy.Builder.newBuilder()
+ .setNumCompactThreads(2)
+ .enableCompactionOnCheckpoint(1)
+ .build(),
+ getTestCommittableSerializer(),
+ compactor,
+ createTestBucketWriter());
+ }
+
+ private BucketWriter<?, String> createTestBucketWriter() {
+ return new BucketWriter<Integer, String>() {
+
+ @Override
+ public InProgressFileWriter<Integer, String> openNewInProgressFile(
+ String bucketId, Path path, long creationTime) throws IOException {
+ return new InProgressFileWriter<Integer, String>() {
+ BufferedWriter writer;
+ long size = 0L;
+
+ @Override
+ public void write(Integer element, long currentTime) throws IOException {
+ if (writer == null) {
+ writer = new BufferedWriter(new FileWriter(path.toString()));
+ }
+ writer.write(element);
+ size += 1;
+ }
+
+ @Override
+ public InProgressFileRecoverable persist() throws IOException {
+ return new TestInProgressFileRecoverable(path, size);
+ }
+
+ @Override
+ public PendingFileRecoverable closeForCommit() throws IOException {
+ return new TestPendingFileRecoverable(path, size);
+ }
+
+ @Override
+ public void dispose() {}
+
+ @Override
+ public String getBucketId() {
+ return bucketId;
+ }
+
+ @Override
+ public long getCreationTime() {
+ return 0;
+ }
+
+ @Override
+ public long getSize() throws IOException {
+ return size;
+ }
+
+ @Override
+ public long getLastUpdateTime() {
+ return 0;
+ }
+ };
+ }
+
+ @Override
+ public InProgressFileWriter<Integer, String> resumeInProgressFileFrom(
+ String s, InProgressFileRecoverable inProgressFileSnapshot, long creationTime)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public WriterProperties getProperties() {
+ return null;
+ }
+
+ @Override
+ public PendingFile recoverPendingFile(PendingFileRecoverable pendingFileRecoverable)
+ throws IOException {
+ return new PendingFile() {
+ @Override
+ public void commit() throws IOException {
+ TestPendingFileRecoverable testRecoverable =
+ (TestPendingFileRecoverable) pendingFileRecoverable;
+ if (testRecoverable.getPath() != null) {
+ if (!testRecoverable
+ .getPath()
+ .equals(testRecoverable.getUncommittedPath())) {
+ testRecoverable
+ .getPath()
+ .getFileSystem()
+ .rename(
+ testRecoverable.getUncommittedPath(),
+ testRecoverable.getPath());
+ }
+ }
+ }
+
+ @Override
+ public void commitAfterRecovery() throws IOException {
+ commit();
+ }
+ };
+ }
+
+ @Override
+ public boolean cleanupInProgressFileRecoverable(
+ InProgressFileRecoverable inProgressFileRecoverable) throws IOException {
+ return false;
+ }
+
+ @Override
+ public CompactingFileWriter openNewCompactingFile(
+ CompactingFileWriter.Type type, String bucketId, Path path, long creationTime)
+ throws IOException {
+ if (type == CompactingFileWriter.Type.RECORD_WISE) {
+ return openNewInProgressFile(bucketId, path, creationTime);
+ } else {
+ FileOutputStream fileOutputStream = new FileOutputStream(path.toString());
+ return new OutputStreamBasedCompactingFileWriter() {
+
+ @Override
+ public OutputStream asOutputStream() throws IOException {
+ return fileOutputStream;
+ }
+
+ @Override
+ public PendingFileRecoverable closeForCommit() throws IOException {
+ fileOutputStream.flush();
+ return new TestPendingFileRecoverable(
+ path, fileOutputStream.getChannel().position());
+ }
+ };
+ }
+ }
+ };
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestTypeInfoTest.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestTypeInfoTest.java
new file mode 100644
index 0000000..c7427b8
--- /dev/null
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/compactor/operator/CompactorRequestTypeInfoTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.file.sink.compactor.operator;
+
+import org.apache.flink.api.common.typeutils.TypeInformationTestBase;
+import org.apache.flink.connector.file.sink.FileSinkCommittableSerializer;
+import org.apache.flink.connector.file.sink.utils.FileSinkTestUtils;
+
+/** Test for {@link CompactorRequestTypeInfo}. */
+public class CompactorRequestTypeInfoTest
+ extends TypeInformationTestBase<CompactorRequestTypeInfo> {
+
+ @Override
+ protected CompactorRequestTypeInfo[] getTestData() {
+ return new CompactorRequestTypeInfo[] {
+ new CompactorRequestTypeInfo(
+ () ->
+ new FileSinkCommittableSerializer(
+ new FileSinkTestUtils.SimpleVersionedWrapperSerializer<>(
+ FileSinkTestUtils.TestPendingFileRecoverable::new),
+ new FileSinkTestUtils.SimpleVersionedWrapperSerializer<>(
+ FileSinkTestUtils.TestInProgressFileRecoverable::new)))
+ };
+ }
+}
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/FileSinkTestUtils.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/FileSinkTestUtils.java
index b3d7041..b5b0c3e 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/FileSinkTestUtils.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/FileSinkTestUtils.java
@@ -36,31 +36,99 @@ public class FileSinkTestUtils {
/** A type of testing {@link InProgressFileWriter.PendingFileRecoverable}. */
public static class TestPendingFileRecoverable extends StringValue
implements InProgressFileWriter.PendingFileRecoverable {
+ private Path path;
+ private Path uncommittedPath;
+ private long size;
+
+ public TestPendingFileRecoverable() {
+ this.path = null;
+ this.uncommittedPath = null;
+ this.size = -1L;
+ }
+
+ public TestPendingFileRecoverable(Path path, long size) {
+ this.path = path;
+ this.uncommittedPath = new Path(path.getParent(), "." + path.getName());
+ this.size = size;
+ }
+
@Override
public Path getPath() {
- return null;
+ return path;
+ }
+
+ public Path getUncommittedPath() {
+ return uncommittedPath;
}
@Override
public long getSize() {
- return -1L;
+ return size;
+ }
+
+ @Override
+ public String getValue() {
+ return size + "," + (path == null ? "" : path.toUri().toString());
+ }
+
+ @Override
+ public void setValue(CharSequence value, int offset, int len) {
+ String[] arr = value.subSequence(offset, len).toString().split(",");
+ size = Integer.parseInt(arr[0]);
+ path = arr.length == 1 ? null : new Path(arr[1]);
+ if (path != null) {
+ uncommittedPath = new Path(path.getParent(), "." + path.getName());
+ }
}
- // Nope
}
/** A type of testing {@link InProgressFileWriter.InProgressFileRecoverable}. */
public static class TestInProgressFileRecoverable extends StringValue
implements InProgressFileWriter.InProgressFileRecoverable {
+ private Path path;
+ private Path uncommittedPath;
+ private long size;
+
+ public TestInProgressFileRecoverable() {
+ this.path = null;
+ this.uncommittedPath = null;
+ this.size = -1L;
+ }
+
+ public TestInProgressFileRecoverable(Path path, long size) {
+ this.path = path;
+ this.uncommittedPath = new Path(path.getParent(), "." + path.getName());
+ this.size = size;
+ }
+
@Override
public Path getPath() {
- return null;
+ return path;
+ }
+
+ public Path getUncommittedPath() {
+ return uncommittedPath;
}
@Override
public long getSize() {
- return -1L;
+ return size;
+ }
+
+ @Override
+ public String getValue() {
+ return size + "," + (path == null ? "" : path.toUri().toString());
+ }
+
+ @Override
+ public void setValue(CharSequence value, int offset, int len) {
+ String[] arr = value.subSequence(offset, len).toString().split(",");
+ size = Integer.parseInt(arr[0]);
+ path = arr.length == 1 ? null : new Path(arr[1]);
+ if (path != null) {
+ uncommittedPath = new Path(path.getParent(), "." + path.getName());
+ }
}
- // Nope
}
/**
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java
index 94d4736..c00cca1 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/utils/IntegerFileSinkTestDataUtils.java
@@ -19,6 +19,7 @@
package org.apache.flink.connector.file.sink.utils;
import org.apache.flink.api.common.serialization.Encoder;
+import org.apache.flink.connector.file.sink.compactor.DecoderBasedReader.Decoder;
import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.streaming.api.functions.sink.filesystem.BucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.SimpleVersionedStringSerializer;
@@ -28,6 +29,7 @@ import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
@@ -56,6 +58,30 @@ public class IntegerFileSinkTestDataUtils {
}
}
+ /** Testing sink {@link Decoder} that reads integer for compaction. */
+ public static class IntDecoder implements Decoder<Integer> {
+
+ private InputStream input;
+
+ @Override
+ public void open(InputStream input) throws IOException {
+ this.input = input;
+ }
+
+ @Override
+ public Integer decodeNext() throws IOException {
+ byte[] bytes = new byte[4];
+ int read = input.read(bytes);
+ return read < 0 ? null : ByteBuffer.wrap(bytes).getInt();
+ }
+
+ @Override
+ public void close() throws IOException {
+ input.close();
+ input = null;
+ }
+ }
+
/** Testing {@link BucketAssigner} that assigns integers according to modulo. */
public static class ModuloBucketAssigner implements BucketAssigner<Integer, String> {
diff --git a/flink-core/src/main/java/org/apache/flink/util/CloseShieldOutputStream.java b/flink-core/src/main/java/org/apache/flink/util/CloseShieldOutputStream.java
new file mode 100644
index 0000000..b0ef6bb
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/CloseShieldOutputStream.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/** A proxy output stream that prevents the underlying output stream from being closed. */
+public class CloseShieldOutputStream extends OutputStream {
+ private final OutputStream out;
+
+ public CloseShieldOutputStream(OutputStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] buffer) throws IOException {
+ out.write(buffer);
+ }
+
+ @Override
+ public void write(byte[] buffer, int off, int len) throws IOException {
+ out.write(buffer, off, len);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ // Do not actually close the internal stream.
+ }
+}
diff --git a/flink-core/src/main/java/org/apache/flink/util/function/SerializableSupplierWithException.java b/flink-core/src/main/java/org/apache/flink/util/function/SerializableSupplierWithException.java
new file mode 100644
index 0000000..ed17baa
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/function/SerializableSupplierWithException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util.function;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.io.Serializable;
+
+/**
+ * A serializable {@link SupplierWithException}.
+ *
+ * @param <T> the type of results supplied by this supplier
+ * @param <E> the type of Exceptions thrown by this function.
+ */
+@PublicEvolving
+@FunctionalInterface
+public interface SerializableSupplierWithException<T, E extends Throwable>
+ extends SupplierWithException<T, E>, Serializable {}
diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroWriterFactory.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroWriterFactory.java
index 473c68e..9475c13 100644
--- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroWriterFactory.java
+++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroWriterFactory.java
@@ -20,11 +20,11 @@ package org.apache.flink.formats.avro;
import org.apache.flink.api.common.serialization.BulkWriter;
import org.apache.flink.core.fs.FSDataOutputStream;
+import org.apache.flink.util.CloseShieldOutputStream;
import org.apache.avro.file.DataFileWriter;
import java.io.IOException;
-import java.io.OutputStream;
/**
* A factory that creates an {@link AvroBulkWriter}. The factory takes a user-supplied builder to
@@ -47,39 +47,4 @@ public class AvroWriterFactory<T> implements BulkWriter.Factory<T> {
public BulkWriter<T> create(FSDataOutputStream out) throws IOException {
return new AvroBulkWriter<>(avroBuilder.createWriter(new CloseShieldOutputStream(out)));
}
-
- /** Proxy output stream that prevents the underlying output stream from being closed. */
- private static class CloseShieldOutputStream extends OutputStream {
- private final OutputStream out;
-
- public CloseShieldOutputStream(OutputStream out) {
- this.out = out;
- }
-
- @Override
- public void write(int b) throws IOException {
- out.write(b);
- }
-
- @Override
- public void write(byte[] buffer) throws IOException {
- out.write(buffer);
- }
-
- @Override
- public void write(byte[] buffer, int off, int len) throws IOException {
- out.write(buffer, off, len);
- }
-
- @Override
- public void flush() throws IOException {
- out.flush();
- }
-
- @Override
- public void close() throws IOException {
- // we do not actually close the internal stream here to prevent that the finishing
- // of the Avro Writer closes the target output stream.
- }
- }
}
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index f5a83ae..1acd328 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -53,6 +53,14 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-files</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
<scope>test</scope>