You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/06/28 04:00:00 UTC

(paimon) branch master updated: [flink] Change the access modifier of sink and operators before sink (#3617)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 29b803dfb [flink] Change the access modifier of sink and operators before sink (#3617)
29b803dfb is described below

commit 29b803dfb527928a94a917e1a75aad43cfba955d
Author: yunfengzhou-hub <yu...@outlook.com>
AuthorDate: Fri Jun 28 11:59:54 2024 +0800

    [flink] Change the access modifier of sink and operators before sink (#3617)
---
 .../flink/sink/DynamicBucketCompactSink.java       |  2 +-
 .../paimon/flink/sink/DynamicBucketSink.java       | 13 +++++++-
 .../apache/paimon/flink/sink/FlinkSinkBuilder.java | 20 ++++++++----
 .../paimon/flink/sink/FlinkTableSinkBase.java      |  6 +++-
 .../apache/paimon/flink/sink/MapToInternalRow.java | 38 ----------------------
 5 files changed, 32 insertions(+), 47 deletions(-)

diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java
index 147e7527f..f9dbd1974 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketCompactSink.java
@@ -50,7 +50,7 @@ public class DynamicBucketCompactSink extends RowDynamicBucketSink {
 
         // bucket-assigner
         HashBucketAssignerOperator<InternalRow> assignerOperator =
-                new HashBucketAssignerOperator<>(
+                createHashBucketAssignerOperator(
                         initialCommitUser, table, null, extractorFunction(), true);
         TupleTypeInfo<Tuple2<InternalRow, Integer>> rowWithBucketType =
                 new TupleTypeInfo<>(input.getType(), BasicTypeInfo.INT_TYPE_INFO);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
index f04043ce4..cf697108f 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java
@@ -20,6 +20,7 @@ package org.apache.paimon.flink.sink;
 
 import org.apache.paimon.schema.TableSchema;
 import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
 import org.apache.paimon.table.sink.ChannelComputer;
 import org.apache.paimon.table.sink.PartitionKeyExtractor;
 import org.apache.paimon.utils.SerializableFunction;
@@ -54,6 +55,16 @@ public abstract class DynamicBucketSink<T> extends FlinkWriteSink<Tuple2<T, Inte
     protected abstract SerializableFunction<TableSchema, PartitionKeyExtractor<T>>
             extractorFunction();
 
+    protected HashBucketAssignerOperator<T> createHashBucketAssignerOperator(
+            String commitUser,
+            Table table,
+            Integer numAssigners,
+            SerializableFunction<TableSchema, PartitionKeyExtractor<T>> extractorFunction,
+            boolean overwrite) {
+        return new HashBucketAssignerOperator<>(
+                commitUser, table, numAssigners, extractorFunction, overwrite);
+    }
+
     public DataStreamSink<?> build(DataStream<T> input, @Nullable Integer parallelism) {
         String initialCommitUser = createCommitUser(table.coreOptions().toConfiguration());
 
@@ -73,7 +84,7 @@ public abstract class DynamicBucketSink<T> extends FlinkWriteSink<Tuple2<T, Inte
 
         // 2. bucket-assigner
         HashBucketAssignerOperator<T> assignerOperator =
-                new HashBucketAssignerOperator<>(
+                createHashBucketAssignerOperator(
                         initialCommitUser, table, numAssigners, extractorFunction(), false);
         TupleTypeInfo<Tuple2<T, Integer>> rowWithBucketType =
                 new TupleTypeInfo<>(partitionByKeyHash.getType(), BasicTypeInfo.INT_TYPE_INFO);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
index 8baee5ac1..dd73bd559 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
@@ -21,6 +21,7 @@ package org.apache.paimon.flink.sink;
 import org.apache.paimon.annotation.Public;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.flink.FlinkConnectorOptions;
+import org.apache.paimon.flink.FlinkRowWrapper;
 import org.apache.paimon.flink.sink.index.GlobalDynamicBucketSink;
 import org.apache.paimon.flink.sorter.TableSortInfo;
 import org.apache.paimon.flink.sorter.TableSorter;
@@ -70,11 +71,11 @@ public class FlinkSinkBuilder {
 
     private static final Logger LOG = LoggerFactory.getLogger(FlinkSinkBuilder.class);
 
-    private final FileStoreTable table;
+    protected final FileStoreTable table;
 
     private DataStream<RowData> input;
-    @Nullable private Map<String, String> overwritePartition;
-    @Nullable private Integer parallelism;
+    @Nullable protected Map<String, String> overwritePartition;
+    @Nullable protected Integer parallelism;
     private Boolean boundedInput = null;
     @Nullable private TableSortInfo tableSortInfo;
 
@@ -221,7 +222,7 @@ public class FlinkSinkBuilder {
     /** Build {@link DataStreamSink}. */
     public DataStreamSink<?> build() {
         input = trySortInput(input);
-        DataStream<InternalRow> input = MapToInternalRow.map(this.input, table.rowType());
+        DataStream<InternalRow> input = mapToInternalRow(this.input, table.rowType());
         if (table.coreOptions().localMergeEnabled() && table.schema().primaryKeys().size() > 0) {
             input =
                     input.forward()
@@ -247,7 +248,14 @@ public class FlinkSinkBuilder {
         }
     }
 
-    private DataStreamSink<?> buildDynamicBucketSink(
+    protected DataStream<InternalRow> mapToInternalRow(
+            DataStream<RowData> input, org.apache.paimon.types.RowType rowType) {
+        return input.map((MapFunction<RowData, InternalRow>) FlinkRowWrapper::new)
+                .setParallelism(input.getParallelism())
+                .returns(org.apache.paimon.flink.utils.InternalTypeInfo.fromRowType(rowType));
+    }
+
+    protected DataStreamSink<?> buildDynamicBucketSink(
             DataStream<InternalRow> input, boolean globalIndex) {
         checkArgument(logSinkFunction == null, "Dynamic bucket mode can not work with log system.");
         return compactSink && !globalIndex
@@ -260,7 +268,7 @@ public class FlinkSinkBuilder {
                                 .build(input, parallelism);
     }
 
-    private DataStreamSink<?> buildForFixedBucket(DataStream<InternalRow> input) {
+    protected DataStreamSink<?> buildForFixedBucket(DataStream<InternalRow> input) {
         DataStream<InternalRow> partitioned =
                 partition(
                         input,
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java
index 4202717c8..ebe1c4c51 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkTableSinkBase.java
@@ -129,7 +129,7 @@ public abstract class FlinkTableSinkBase
                 overwrite ? null : (logSinkProvider == null ? null : logSinkProvider.createSink());
         return new PaimonDataStreamSinkProvider(
                 (dataStream) -> {
-                    LogFlinkSinkBuilder builder = new LogFlinkSinkBuilder(table);
+                    LogFlinkSinkBuilder builder = createSinkBuilder();
                     builder.logSinkFunction(logSinkFunction)
                             .forRowData(
                                     new DataStream<>(
@@ -149,6 +149,10 @@ public abstract class FlinkTableSinkBase
                 });
     }
 
+    protected LogFlinkSinkBuilder createSinkBuilder() {
+        return new LogFlinkSinkBuilder(table);
+    }
+
     @Override
     public DynamicTableSink copy() {
         FlinkTableSink copied =
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MapToInternalRow.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MapToInternalRow.java
deleted file mode 100644
index a9bf74401..000000000
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MapToInternalRow.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.flink.sink;
-
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.flink.FlinkRowWrapper;
-import org.apache.paimon.flink.utils.InternalTypeInfo;
-import org.apache.paimon.types.RowType;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.table.data.RowData;
-
-/** An util to convert {@link RowData} stream to {@link InternalRow} stream. */
-public class MapToInternalRow {
-
-    public static DataStream<InternalRow> map(DataStream<RowData> input, RowType rowType) {
-        return input.map((MapFunction<RowData, InternalRow>) FlinkRowWrapper::new)
-                .setParallelism(input.getParallelism())
-                .returns(InternalTypeInfo.fromRowType(rowType));
-    }
-}