You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/06/28 04:00:00 UTC
(paimon) branch master updated: [flink] Change the access modifier of sink and operators before sink (#3617)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 29b803dfb [flink] Change the access modifier of sink and operators before sink (#3617)
29b803dfb is described below
commit 29b803dfb527928a94a917e1a75aad43cfba955d
Author: yunfengzhou-hub <yu...@outlook.com>
AuthorDate: Fri Jun 28 11:59:54 2024 +0800
[flink] Change the access modifier of sink and operators before sink (#3617)
---
.../flink/sink/DynamicBucketCompactSink.java | 2 +-
.../paimon/flink/sink/DynamicBucketSink.java | 13 +++++++-
.../apache/paimon/flink/sink/FlinkSinkBuilder.java | 20 ++++++++----
.../paimon/flink/sink/FlinkTableSinkBase.java | 6 +++-
.../apache/paimon/flink/sink/MapToInternalRow.java | 38 ----------------------
5 files changed, 32 insertions(+), 47 deletions(-)
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java
index 147e7527f..f9dbd1974 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java
@@ -50,7 +50,7 @@ public class DynamicBucketCompactSink extends RowDynamicBucketSink {
// bucket-assigner
HashBucketAssignerOperator<InternalRow> assignerOperator =
- new HashBucketAssignerOperator<>(
+ createHashBucketAssignerOperator(
initialCommitUser, table, null, extractorFunction(), true);
TupleTypeInfo<Tuple2<InternalRow, Integer>> rowWithBucketType =
new TupleTypeInfo<>(input.getType(), BasicTypeInfo.INT_TYPE_INFO);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
index f04043ce4..cf697108f 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
@@ -20,6 +20,7 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
import org.apache.paimon.table.sink.ChannelComputer;
import org.apache.paimon.table.sink.PartitionKeyExtractor;
import org.apache.paimon.utils.SerializableFunction;
@@ -54,6 +55,16 @@ public abstract class DynamicBucketSink<T> extends FlinkWriteSink<Tuple2<T, Inte
protected abstract SerializableFunction<TableSchema, PartitionKeyExtractor<T>>
extractorFunction();
+ protected HashBucketAssignerOperator<T> createHashBucketAssignerOperator(
+ String commitUser,
+ Table table,
+ Integer numAssigners,
+ SerializableFunction<TableSchema, PartitionKeyExtractor<T>> extractorFunction,
+ boolean overwrite) {
+ return new HashBucketAssignerOperator<>(
+ commitUser, table, numAssigners, extractorFunction, overwrite);
+ }
+
public DataStreamSink<?> build(DataStream<T> input, @Nullable Integer parallelism) {
String initialCommitUser = createCommitUser(table.coreOptions().toConfiguration());
@@ -73,7 +84,7 @@ public abstract class DynamicBucketSink<T> extends FlinkWriteSink<Tuple2<T, Inte
// 2. bucket-assigner
HashBucketAssignerOperator<T> assignerOperator =
- new HashBucketAssignerOperator<>(
+ createHashBucketAssignerOperator(
initialCommitUser, table, numAssigners, extractorFunction(), false);
TupleTypeInfo<Tuple2<T, Integer>> rowWithBucketType =
new TupleTypeInfo<>(partitionByKeyHash.getType(), BasicTypeInfo.INT_TYPE_INFO);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
index 8baee5ac1..dd73bd559 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
@@ -21,6 +21,7 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.annotation.Public;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.flink.FlinkConnectorOptions;
+import org.apache.paimon.flink.FlinkRowWrapper;
import org.apache.paimon.flink.sink.index.GlobalDynamicBucketSink;
import org.apache.paimon.flink.sorter.TableSortInfo;
import org.apache.paimon.flink.sorter.TableSorter;
@@ -70,11 +71,11 @@ public class FlinkSinkBuilder {
private static final Logger LOG = LoggerFactory.getLogger(FlinkSinkBuilder.class);
- private final FileStoreTable table;
+ protected final FileStoreTable table;
private DataStream<RowData> input;
- @Nullable private Map<String, String> overwritePartition;
- @Nullable private Integer parallelism;
+ @Nullable protected Map<String, String> overwritePartition;
+ @Nullable protected Integer parallelism;
private Boolean boundedInput = null;
@Nullable private TableSortInfo tableSortInfo;
@@ -221,7 +222,7 @@ public class FlinkSinkBuilder {
/** Build {@link DataStreamSink}. */
public DataStreamSink<?> build() {
input = trySortInput(input);
- DataStream<InternalRow> input = MapToInternalRow.map(this.input, table.rowType());
+ DataStream<InternalRow> input = mapToInternalRow(this.input, table.rowType());
if (table.coreOptions().localMergeEnabled() && table.schema().primaryKeys().size() > 0) {
input =
input.forward()
@@ -247,7 +248,14 @@ public class FlinkSinkBuilder {
}
}
- private DataStreamSink<?> buildDynamicBucketSink(
+ protected DataStream<InternalRow> mapToInternalRow(
+ DataStream<RowData> input, org.apache.paimon.types.RowType rowType) {
+ return input.map((MapFunction<RowData, InternalRow>) FlinkRowWrapper::new)
+ .setParallelism(input.getParallelism())
+ .returns(org.apache.paimon.flink.utils.InternalTypeInfo.fromRowType(rowType));
+ }
+
+ protected DataStreamSink<?> buildDynamicBucketSink(
DataStream<InternalRow> input, boolean globalIndex) {
checkArgument(logSinkFunction == null, "Dynamic bucket mode can not work with log system.");
return compactSink && !globalIndex
@@ -260,7 +268,7 @@ public class FlinkSinkBuilder {
.build(input, parallelism);
}
- private DataStreamSink<?> buildForFixedBucket(DataStream<InternalRow> input) {
+ protected DataStreamSink<?> buildForFixedBucket(DataStream<InternalRow> input) {
DataStream<InternalRow> partitioned =
partition(
input,
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java
index 4202717c8..ebe1c4c51 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java
@@ -129,7 +129,7 @@ public abstract class FlinkTableSinkBase
overwrite ? null : (logSinkProvider == null ? null : logSinkProvider.createSink());
return new PaimonDataStreamSinkProvider(
(dataStream) -> {
- LogFlinkSinkBuilder builder = new LogFlinkSinkBuilder(table);
+ LogFlinkSinkBuilder builder = createSinkBuilder();
builder.logSinkFunction(logSinkFunction)
.forRowData(
new DataStream<>(
@@ -149,6 +149,10 @@ public abstract class FlinkTableSinkBase
});
}
+ protected LogFlinkSinkBuilder createSinkBuilder() {
+ return new LogFlinkSinkBuilder(table);
+ }
+
@Override
public DynamicTableSink copy() {
FlinkTableSink copied =
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MapToInternalRow.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MapToInternalRow.java
deleted file mode 100644
index a9bf74401..000000000
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MapToInternalRow.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.flink.sink;
-
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.flink.FlinkRowWrapper;
-import org.apache.paimon.flink.utils.InternalTypeInfo;
-import org.apache.paimon.types.RowType;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.table.data.RowData;
-
-/** An util to convert {@link RowData} stream to {@link InternalRow} stream. */
-public class MapToInternalRow {
-
- public static DataStream<InternalRow> map(DataStream<RowData> input, RowType rowType) {
- return input.map((MapFunction<RowData, InternalRow>) FlinkRowWrapper::new)
- .setParallelism(input.getParallelism())
- .returns(InternalTypeInfo.fromRowType(rowType));
- }
-}