You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2022/12/13 22:11:48 UTC
[hudi] 07/10: [HUDI-5342] Add new bulk insert sort modes repartitioning data by partition path (#7402)
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.12.2-shadow
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit e4bdb733bcbc44fab6cf2a48349137bfc9936ce9
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri Dec 9 19:04:44 2022 -0800
[HUDI-5342] Add new bulk insert sort modes repartitioning data by partition path (#7402)
This PR adds two new bulk insert sort modes, PARTITION_PATH_REPARTITION and PARTITION_PATH_REPARTITION_AND_SORT, which does the following
For a physically partitioned table, repartition the input records based on the partition path, limiting the shuffle parallelism to specified outputSparkPartitions. For PARTITION_PATH_REPARTITION_AND_SORT, an additional step of sorting the records based on the partition path within each Spark partition is done.
For a physically non-partitioned table, simply does coalesce for the input rows with outputSparkPartitions.
New unit tests are added to verify the added functionality.
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 11 ++++
.../execution/bulkinsert/BulkInsertSortMode.java | 8 ++-
.../java/org/apache/hudi/table/HoodieTable.java | 7 +++
.../MultipleSparkJobExecutionStrategy.java | 13 ++--
.../BulkInsertInternalPartitionerFactory.java | 12 +++-
...lkInsertInternalPartitionerWithRowsFactory.java | 7 ++-
.../bulkinsert/PartitionPathRDDPartitioner.java | 52 ++++++++++++++++
...PartitionPathRepartitionAndSortPartitioner.java | 71 ++++++++++++++++++++++
...nPathRepartitionAndSortPartitionerWithRows.java | 62 +++++++++++++++++++
.../PartitionPathRepartitionPartitioner.java | 70 +++++++++++++++++++++
...artitionPathRepartitionPartitionerWithRows.java | 60 ++++++++++++++++++
.../table/action/commit/SparkBulkInsertHelper.java | 2 +-
.../TestBulkInsertInternalPartitioner.java | 20 +++---
.../TestBulkInsertInternalPartitionerForRows.java | 24 +++++---
.../function/SerializableFunctionUnchecked.java | 34 +++++++++++
.../hudi/common/table/HoodieTableConfig.java | 4 ++
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 8 ++-
17 files changed, 436 insertions(+), 29 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 533bb5c47df..9b0371008d1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -317,6 +317,17 @@ public class HoodieWriteConfig extends HoodieConfig {
+ "GLOBAL_SORT: this ensures best file sizes, with lowest memory overhead at cost of sorting. "
+ "PARTITION_SORT: Strikes a balance by only sorting within a partition, still keeping the memory overhead of writing "
+ "lowest and best effort file sizing. "
+ + "PARTITION_PATH_REPARTITION: this ensures that the data for a single physical partition in the table is written "
+ + "by the same Spark executor, best for input data evenly distributed across different partition paths. "
+ + "This can cause imbalance among Spark executors if the input data is skewed, i.e., most records are intended for "
+ + "a handful of partition paths among all. "
+ + "PARTITION_PATH_REPARTITION_AND_SORT: this ensures that the data for a single physical partition in the table is written "
+ + "by the same Spark executor, best for input data evenly distributed across different partition paths. "
+ + "Compared to PARTITION_PATH_REPARTITION, this sort mode does an additional step of sorting the records "
+ + "based on the partition path within a single Spark partition, given that data for multiple physical partitions "
+ + "can be sent to the same Spark partition and executor. "
+ + "This can cause imbalance among Spark executors if the input data is skewed, i.e., most records are intended for "
+ + "a handful of partition paths among all. "
+ "NONE: No sorting. Fastest and matches `spark.write.parquet()` in terms of number of files, overheads");
public static final ConfigProperty<String> EMBEDDED_TIMELINE_SERVER_ENABLE = ConfigProperty
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
index d171b8cd726..6fd06545ae5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
@@ -22,7 +22,9 @@ package org.apache.hudi.execution.bulkinsert;
* Bulk insert sort mode.
*/
public enum BulkInsertSortMode {
- NONE,
- GLOBAL_SORT,
- PARTITION_SORT
+ NONE,
+ GLOBAL_SORT,
+ PARTITION_SORT,
+ PARTITION_PATH_REPARTITION,
+ PARTITION_PATH_REPARTITION_AND_SORT
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index b09d488e7d2..77b24a38953 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -279,6 +279,13 @@ public abstract class HoodieTable<T extends HoodieRecordPayload, I, K, O> implem
return metaClient;
}
+ /**
+ * @return if the table is physically partitioned, based on the partition fields stored in the table config.
+ */
+ public boolean isPartitioned() {
+ return getMetaClient().getTableConfig().isTablePartitioned();
+ }
+
public Configuration getHadoopConf() {
return metaClient.getHadoopConf();
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
index 4a1823939fb..15a72079910 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
@@ -187,7 +187,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T extends HoodieRecordPa
Option<String[]> orderByColumnsOpt =
Option.ofNullable(strategyParams.get(PLAN_STRATEGY_SORT_COLUMNS.key()))
.map(listStr -> listStr.split(","));
-
+
return orderByColumnsOpt.map(orderByColumns -> {
HoodieClusteringConfig.LayoutOptimizationStrategy layoutOptStrategy = getWriteConfig().getLayoutOptimizationStrategy();
switch (layoutOptStrategy) {
@@ -196,17 +196,20 @@ public abstract class MultipleSparkJobExecutionStrategy<T extends HoodieRecordPa
return isRowPartitioner
? new RowSpatialCurveSortPartitioner(getWriteConfig())
: new RDDSpatialCurveSortPartitioner((HoodieSparkEngineContext) getEngineContext(), orderByColumns, layoutOptStrategy,
- getWriteConfig().getLayoutOptimizationCurveBuildMethod(), HoodieAvroUtils.addMetadataFields(schema));
+ getWriteConfig().getLayoutOptimizationCurveBuildMethod(), HoodieAvroUtils.addMetadataFields(schema));
case LINEAR:
return isRowPartitioner
? new RowCustomColumnsSortPartitioner(orderByColumns)
: new RDDCustomColumnsSortPartitioner(orderByColumns, HoodieAvroUtils.addMetadataFields(schema),
- getWriteConfig().isConsistentLogicalTimestampEnabled());
+ getWriteConfig().isConsistentLogicalTimestampEnabled());
default:
throw new UnsupportedOperationException(String.format("Layout optimization strategy '%s' is not supported", layoutOptStrategy));
}
- }).orElse(isRowPartitioner ? BulkInsertInternalPartitionerWithRowsFactory.get(getWriteConfig().getBulkInsertSortMode()) :
- BulkInsertInternalPartitionerFactory.get(getWriteConfig().getBulkInsertSortMode()));
+ }).orElse(isRowPartitioner
+ ? BulkInsertInternalPartitionerWithRowsFactory.get(
+ getWriteConfig().getBulkInsertSortMode(), getHoodieTable().isPartitioned())
+ : BulkInsertInternalPartitionerFactory.get(
+ getWriteConfig().getBulkInsertSortMode(), getHoodieTable().isPartitioned()));
}
/**
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
index aaa7b5b4bfa..84bd79e3a22 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
@@ -18,8 +18,10 @@
package org.apache.hudi.execution.bulkinsert;
+import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.table.BulkInsertPartitioner;
+import org.apache.hudi.table.HoodieTable;
/**
* A factory to generate built-in partitioner to repartition input records into at least
@@ -27,7 +29,11 @@ import org.apache.hudi.table.BulkInsertPartitioner;
*/
public abstract class BulkInsertInternalPartitionerFactory {
- public static BulkInsertPartitioner get(BulkInsertSortMode sortMode) {
+ public static BulkInsertPartitioner get(HoodieTable table, HoodieWriteConfig config) {
+ return get(config.getBulkInsertSortMode(), table.isPartitioned());
+ }
+
+ public static BulkInsertPartitioner get(BulkInsertSortMode sortMode, boolean isTablePartitioned) {
switch (sortMode) {
case NONE:
return new NonSortPartitioner();
@@ -35,6 +41,10 @@ public abstract class BulkInsertInternalPartitionerFactory {
return new GlobalSortPartitioner();
case PARTITION_SORT:
return new RDDPartitionSortPartitioner();
+ case PARTITION_PATH_REPARTITION:
+ return new PartitionPathRepartitionPartitioner(isTablePartitioned);
+ case PARTITION_PATH_REPARTITION_AND_SORT:
+ return new PartitionPathRepartitionAndSortPartitioner(isTablePartitioned);
default:
throw new HoodieException("The bulk insert sort mode \"" + sortMode.name() + "\" is not supported.");
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerWithRowsFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerWithRowsFactory.java
index 47b3efbb77c..720c03948d4 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerWithRowsFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerWithRowsFactory.java
@@ -29,7 +29,8 @@ import org.apache.spark.sql.Row;
*/
public abstract class BulkInsertInternalPartitionerWithRowsFactory {
- public static BulkInsertPartitioner<Dataset<Row>> get(BulkInsertSortMode sortMode) {
+ public static BulkInsertPartitioner<Dataset<Row>> get(BulkInsertSortMode sortMode,
+ boolean isTablePartitioned) {
switch (sortMode) {
case NONE:
return new NonSortPartitionerWithRows();
@@ -37,6 +38,10 @@ public abstract class BulkInsertInternalPartitionerWithRowsFactory {
return new GlobalSortPartitionerWithRows();
case PARTITION_SORT:
return new PartitionSortPartitionerWithRows();
+ case PARTITION_PATH_REPARTITION:
+ return new PartitionPathRepartitionPartitionerWithRows(isTablePartitioned);
+ case PARTITION_PATH_REPARTITION_AND_SORT:
+ return new PartitionPathRepartitionAndSortPartitionerWithRows(isTablePartitioned);
default:
throw new UnsupportedOperationException("The bulk insert sort mode \"" + sortMode.name() + "\" is not supported.");
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRDDPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRDDPartitioner.java
new file mode 100644
index 00000000000..eb835b38c34
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRDDPartitioner.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.execution.bulkinsert;
+
+import org.apache.hudi.common.function.SerializableFunctionUnchecked;
+
+import org.apache.spark.Partitioner;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * A Spark RDD partitioner implementation that determines the Spark partition
+ * based on the table partition path, generating targeted number of Spark partitions.
+ */
+public class PartitionPathRDDPartitioner extends Partitioner implements Serializable {
+ private final SerializableFunctionUnchecked<Object, String> partitionPathExtractor;
+ private final int numPartitions;
+
+ PartitionPathRDDPartitioner(SerializableFunctionUnchecked<Object, String> partitionPathExtractor, int numPartitions) {
+ this.partitionPathExtractor = partitionPathExtractor;
+ this.numPartitions = numPartitions;
+ }
+
+ @Override
+ public int numPartitions() {
+ return numPartitions;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int getPartition(Object o) {
+ return Math.abs(Objects.hash(partitionPathExtractor.apply(o))) % numPartitions;
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitioner.java
new file mode 100644
index 00000000000..e8e1e2072f5
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitioner.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.execution.bulkinsert;
+
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.table.BulkInsertPartitioner;
+
+import org.apache.spark.api.java.JavaRDD;
+
+import scala.Tuple2;
+
+/**
+ * A built-in partitioner that does the following for input records for bulk insert operation
+ * <p>
+ * - For physically partitioned table, repartition the input records based on the partition path,
+ * and sort records within Spark partitions, limiting the shuffle parallelism to specified
+ * `outputSparkPartitions`
+ * <p>
+ * - For physically non-partitioned table, simply does coalesce for the input records with
+ * `outputSparkPartitions`
+ * <p>
+ * Corresponding to the {@code BulkInsertSortMode.PARTITION_PATH_REPARTITION_AND_SORT} mode.
+ *
+ * @param <T> HoodieRecordPayload type
+ */
+public class PartitionPathRepartitionAndSortPartitioner<T extends HoodieRecordPayload>
+ implements BulkInsertPartitioner<JavaRDD<HoodieRecord<T>>> {
+
+ private final boolean isTablePartitioned;
+
+ public PartitionPathRepartitionAndSortPartitioner(boolean isTablePartitioned) {
+ this.isTablePartitioned = isTablePartitioned;
+ }
+
+ @Override
+ public JavaRDD<HoodieRecord<T>> repartitionRecords(JavaRDD<HoodieRecord<T>> records,
+ int outputSparkPartitions) {
+ if (isTablePartitioned) {
+ PartitionPathRDDPartitioner partitioner = new PartitionPathRDDPartitioner(
+ (partitionPath) -> (String) partitionPath, outputSparkPartitions);
+ return records
+ .mapToPair(record -> new Tuple2<>(record.getPartitionPath(), record))
+ .repartitionAndSortWithinPartitions(partitioner)
+ .values();
+ }
+ return records.coalesce(outputSparkPartitions);
+ }
+
+ @Override
+ public boolean arePartitionRecordsSorted() {
+ return isTablePartitioned;
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitionerWithRows.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitionerWithRows.java
new file mode 100644
index 00000000000..cf3ff1acfa4
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitionerWithRows.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.execution.bulkinsert;
+
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.table.BulkInsertPartitioner;
+
+import org.apache.spark.sql.Column;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+
+/**
+ * A built-in partitioner that does the following for input rows for bulk insert operation
+ * <p>
+ * - For physically partitioned table, repartition the input rows based on the partition path,
+ * and sort rows within Spark partitions, limiting the shuffle parallelism to specified
+ * `outputSparkPartitions`
+ * <p>
+ * - For physically non-partitioned table, simply does coalesce for the input rows with
+ * `outputSparkPartitions`
+ * <p>
+ * Corresponding to the {@code BulkInsertSortMode.PARTITION_PATH_REPARTITION_AND_SORT} mode.
+ */
+public class PartitionPathRepartitionAndSortPartitionerWithRows implements BulkInsertPartitioner<Dataset<Row>> {
+
+ private final boolean isTablePartitioned;
+
+ public PartitionPathRepartitionAndSortPartitionerWithRows(boolean isTablePartitioned) {
+ this.isTablePartitioned = isTablePartitioned;
+ }
+
+ @Override
+ public Dataset<Row> repartitionRecords(Dataset<Row> rows, int outputSparkPartitions) {
+ if (isTablePartitioned) {
+ return rows.repartition(outputSparkPartitions, new Column(HoodieRecord.PARTITION_PATH_METADATA_FIELD))
+ .sortWithinPartitions(new Column(HoodieRecord.PARTITION_PATH_METADATA_FIELD));
+ }
+ return rows.coalesce(outputSparkPartitions);
+ }
+
+ @Override
+ public boolean arePartitionRecordsSorted() {
+ return isTablePartitioned;
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitioner.java
new file mode 100644
index 00000000000..5931b565757
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitioner.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.execution.bulkinsert;
+
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.table.BulkInsertPartitioner;
+
+import org.apache.spark.api.java.JavaRDD;
+
+import scala.Tuple2;
+
+/**
+ * A built-in partitioner that does the following for input records for bulk insert operation
+ * <p>
+ * - For physically partitioned table, repartition the input records based on the partition path,
+ * limiting the shuffle parallelism to specified `outputSparkPartitions`
+ * <p>
+ * - For physically non-partitioned table, simply does coalesce for the input records with
+ * `outputSparkPartitions`
+ * <p>
+ * Corresponding to the {@code BulkInsertSortMode.PARTITION_PATH_REPARTITION} mode.
+ *
+ * @param <T> HoodieRecordPayload type
+ */
+public class PartitionPathRepartitionPartitioner<T extends HoodieRecordPayload>
+ implements BulkInsertPartitioner<JavaRDD<HoodieRecord<T>>> {
+
+ private final boolean isTablePartitioned;
+
+ public PartitionPathRepartitionPartitioner(boolean isTablePartitioned) {
+ this.isTablePartitioned = isTablePartitioned;
+ }
+
+ @Override
+ public JavaRDD<HoodieRecord<T>> repartitionRecords(JavaRDD<HoodieRecord<T>> records,
+ int outputSparkPartitions) {
+ if (isTablePartitioned) {
+ PartitionPathRDDPartitioner partitioner = new PartitionPathRDDPartitioner(
+ (partitionPath) -> (String) partitionPath, outputSparkPartitions);
+ return records
+ .mapToPair(record -> new Tuple2<>(record.getPartitionPath(), record))
+ .partitionBy(partitioner)
+ .values();
+ }
+ return records.coalesce(outputSparkPartitions);
+ }
+
+ @Override
+ public boolean arePartitionRecordsSorted() {
+ return false;
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitionerWithRows.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitionerWithRows.java
new file mode 100644
index 00000000000..62d9edbca84
--- /dev/null
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitionerWithRows.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.execution.bulkinsert;
+
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.table.BulkInsertPartitioner;
+
+import org.apache.spark.sql.Column;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+
+/**
+ * A built-in partitioner that does the following for input rows for bulk insert operation
+ * <p>
+ * - For physically partitioned table, repartition the input rows based on the partition path,
+ * limiting the shuffle parallelism to specified `outputSparkPartitions`
+ * <p>
+ * - For physically non-partitioned table, simply does coalesce for the input rows with
+ * `outputSparkPartitions`
+ * <p>
+ * Corresponding to the {@code BulkInsertSortMode.PARTITION_PATH_REPARTITION} mode.
+ */
+public class PartitionPathRepartitionPartitionerWithRows implements BulkInsertPartitioner<Dataset<Row>> {
+
+ private final boolean isTablePartitioned;
+
+ public PartitionPathRepartitionPartitionerWithRows(boolean isTablePartitioned) {
+ this.isTablePartitioned = isTablePartitioned;
+ }
+
+ @Override
+ public Dataset<Row> repartitionRecords(Dataset<Row> rows, int outputSparkPartitions) {
+ if (isTablePartitioned) {
+ return rows.repartition(outputSparkPartitions, new Column(HoodieRecord.PARTITION_PATH_METADATA_FIELD));
+ }
+ return rows.coalesce(outputSparkPartitions);
+ }
+
+ @Override
+ public boolean arePartitionRecordsSorted() {
+ return false;
+ }
+}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java
index 1652c35eb63..9f1a7813e8c 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java
@@ -74,7 +74,7 @@ public class SparkBulkInsertHelper<T extends HoodieRecordPayload, R> extends Bas
executor.getCommitActionType(), instantTime), Option.empty(),
config.shouldAllowMultiWriteOnSameInstant());
- BulkInsertPartitioner partitioner = userDefinedBulkInsertPartitioner.orElse(BulkInsertInternalPartitionerFactory.get(config.getBulkInsertSortMode()));
+ BulkInsertPartitioner partitioner = userDefinedBulkInsertPartitioner.orElse(BulkInsertInternalPartitionerFactory.get(table, config));
// write new files
HoodieData<WriteStatus> writeStatuses = bulkInsert(inputRecords, instantTime, table, config, performDedupe, partitioner, false,
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
index 5cb7859888c..d04059e59f9 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
@@ -79,9 +79,13 @@ public class TestBulkInsertInternalPartitioner extends HoodieClientTestBase impl
private static Stream<Arguments> configParams() {
Object[][] data = new Object[][] {
- {BulkInsertSortMode.GLOBAL_SORT, true, true},
- {BulkInsertSortMode.PARTITION_SORT, false, true},
- {BulkInsertSortMode.NONE, false, false}
+ {BulkInsertSortMode.GLOBAL_SORT, true, true, true},
+ {BulkInsertSortMode.PARTITION_SORT, true, false, true},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION, true, false, false},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION, false, false, false},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION_AND_SORT, true, false, false},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION_AND_SORT, false, false, false},
+ {BulkInsertSortMode.NONE, true, false, false}
};
return Stream.of(data).map(Arguments::of);
}
@@ -133,15 +137,17 @@ public class TestBulkInsertInternalPartitioner extends HoodieClientTestBase impl
assertEquals(expectedPartitionNumRecords, actualPartitionNumRecords);
}
- @ParameterizedTest(name = "[{index}] {0}")
+ @ParameterizedTest(name = "[{index}] {0} isTablePartitioned={1}")
@MethodSource("configParams")
public void testBulkInsertInternalPartitioner(BulkInsertSortMode sortMode,
- boolean isGloballySorted, boolean isLocallySorted) {
+ boolean isTablePartitioned,
+ boolean isGloballySorted,
+ boolean isLocallySorted) {
JavaRDD<HoodieRecord> records1 = generateTestRecordsForBulkInsert(jsc);
JavaRDD<HoodieRecord> records2 = generateTripleTestRecordsForBulkInsert(jsc);
- testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerFactory.get(sortMode),
+ testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerFactory.get(sortMode, isTablePartitioned),
records1, isGloballySorted, isLocallySorted, generateExpectedPartitionNumRecords(records1));
- testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerFactory.get(sortMode),
+ testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerFactory.get(sortMode, isTablePartitioned),
records2, isGloballySorted, isLocallySorted, generateExpectedPartitionNumRecords(records2));
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitionerForRows.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitionerForRows.java
index 52cf67228a5..6223c8bb9eb 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitionerForRows.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitionerForRows.java
@@ -67,23 +67,30 @@ public class TestBulkInsertInternalPartitionerForRows extends HoodieClientTestHa
private static Stream<Arguments> configParams() {
Object[][] data = new Object[][] {
- {BulkInsertSortMode.GLOBAL_SORT, true, true},
- {BulkInsertSortMode.PARTITION_SORT, false, true},
- {BulkInsertSortMode.NONE, false, false}
+ {BulkInsertSortMode.GLOBAL_SORT, true, true, true},
+ {BulkInsertSortMode.PARTITION_SORT, true, false, true},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION, true, false, false},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION, false, false, false},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION_AND_SORT, true, false, false},
+ {BulkInsertSortMode.PARTITION_PATH_REPARTITION_AND_SORT, false, false, false},
+
+ {BulkInsertSortMode.NONE, true, false, false}
};
return Stream.of(data).map(Arguments::of);
}
- @ParameterizedTest(name = "[{index}] {0}")
+ @ParameterizedTest(name = "[{index}] {0} isTablePartitioned={1}")
@MethodSource("configParams")
public void testBulkInsertInternalPartitioner(BulkInsertSortMode sortMode,
- boolean isGloballySorted, boolean isLocallySorted)
+ boolean isTablePartitioned,
+ boolean isGloballySorted,
+ boolean isLocallySorted)
throws Exception {
Dataset<Row> records1 = generateTestRecords();
Dataset<Row> records2 = generateTestRecords();
- testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerWithRowsFactory.get(sortMode),
+ testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerWithRowsFactory.get(sortMode, isTablePartitioned),
records1, isGloballySorted, isLocallySorted, generateExpectedPartitionNumRecords(records1), Option.empty());
- testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerWithRowsFactory.get(sortMode),
+ testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerWithRowsFactory.get(sortMode, isTablePartitioned),
records2, isGloballySorted, isLocallySorted, generateExpectedPartitionNumRecords(records2), Option.empty());
}
@@ -158,7 +165,8 @@ public class TestBulkInsertInternalPartitionerForRows extends HoodieClientTestHa
public Dataset<Row> generateTestRecords() {
Dataset<Row> rowsPart1 = SparkDatasetTestUtils.getRandomRows(sqlContext, 100, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, false);
Dataset<Row> rowsPart2 = SparkDatasetTestUtils.getRandomRows(sqlContext, 150, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, false);
- return rowsPart1.union(rowsPart2);
+ Dataset<Row> rowsPart3 = SparkDatasetTestUtils.getRandomRows(sqlContext, 200, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, false);
+ return rowsPart1.union(rowsPart2).union(rowsPart3);
}
private void verifyRowsAscendingOrder(List<Row> records, Option<Comparator<Row>> comparator) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/function/SerializableFunctionUnchecked.java b/hudi-common/src/main/java/org/apache/hudi/common/function/SerializableFunctionUnchecked.java
new file mode 100644
index 00000000000..fd62d033243
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/common/function/SerializableFunctionUnchecked.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.common.function;
+
+import java.io.Serializable;
+import java.util.function.Function;
+
+/**
+ * Serializable {@link Function} interface that only might be throwing unchecked exceptions
+ *
+ * @param <I> input type
+ * @param <O> output type
+ */
+@FunctionalInterface
+public interface SerializableFunctionUnchecked<I, O> extends Serializable {
+ O apply(I v1);
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
index 369eced0312..371bb6dae11 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
@@ -499,6 +499,10 @@ public class HoodieTableConfig extends HoodieConfig {
return Option.empty();
}
+ public boolean isTablePartitioned() {
+ return getPartitionFields().map(pfs -> pfs.length > 0).orElse(false);
+ }
+
/**
* @returns the partition field prop.
* @deprecated please use {@link #getPartitionFields()} instead
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 108bd58972a..51cd30f7f23 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -180,7 +180,7 @@ object HoodieSparkSqlWriter {
if (hoodieConfig.getBoolean(ENABLE_ROW_WRITER) &&
operation == WriteOperationType.BULK_INSERT) {
val (success, commitTime: common.util.Option[String]) = bulkInsertAsRow(sqlContext, parameters, df, tblName,
- basePath, path, instantTime, partitionColumns)
+ basePath, path, instantTime, partitionColumns, tableConfig.isTablePartitioned)
return (success, commitTime, common.util.Option.empty(), common.util.Option.empty(), hoodieWriteClient.orNull, tableConfig)
}
// scalastyle:on
@@ -572,7 +572,8 @@ object HoodieSparkSqlWriter {
basePath: Path,
path: String,
instantTime: String,
- partitionColumns: String): (Boolean, common.util.Option[String]) = {
+ partitionColumns: String,
+ isTablePartitioned: Boolean): (Boolean, common.util.Option[String]) = {
val sparkContext = sqlContext.sparkContext
val populateMetaFields = java.lang.Boolean.parseBoolean(parameters.getOrElse(HoodieTableConfig.POPULATE_META_FIELDS.key(),
String.valueOf(HoodieTableConfig.POPULATE_META_FIELDS.defaultValue())))
@@ -601,7 +602,8 @@ object HoodieSparkSqlWriter {
if (userDefinedBulkInsertPartitionerOpt.isPresent) {
userDefinedBulkInsertPartitionerOpt.get
} else {
- BulkInsertInternalPartitionerWithRowsFactory.get(writeConfig.getBulkInsertSortMode)
+ BulkInsertInternalPartitionerWithRowsFactory.get(
+ writeConfig.getBulkInsertSortMode, isTablePartitioned)
}
} else {
// Sort modes are not yet supported when meta fields are disabled