You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Ethan Guo (Jira)" <ji...@apache.org> on 2022/09/06 05:15:00 UTC
[jira] [Updated] (HUDI-4655) Test COW: Spark datasource metadata-only and full-record bootstrap operation
[ https://issues.apache.org/jira/browse/HUDI-4655?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ethan Guo updated HUDI-4655:
----------------------------
Description:
Testing both Hive-style and date (year/month/day) partitions
{code:java}
import org.apache.hudi.bootstrap.SparkParquetBootstrapDataProvider
import org.apache.hudi.client.bootstrap.selector.BootstrapRegexModeSelector
import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers}
import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieWriteConfig}
import org.apache.hudi.keygen.SimpleKeyGenerator
import org.apache.spark.sql.SaveMode
val srcPath = "<>/Users/ethan/Work/scripts/bootstrap-testing/partitioned-parquet-table"
val basePath = "<>/Users/ethan/Work/scripts/bootstrap-testing/bootstrap-hudi-table-1"
val bootstrapDF = spark.emptyDataFrame
bootstrapDF.write
.format("hudi")
.option(HoodieWriteConfig.TABLE_NAME, "hoodie_test")
.option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL)
.option(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY, "key")
.option(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY, "partition")
.option(DataSourceWriteOptions.PRECOMBINE_FIELD_OPT_KEY, "ts")
.option(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH_PROP, srcPath)
.option(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS, classOf[SimpleKeyGenerator].getName)
.option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR, classOf[BootstrapRegexModeSelector].getName)
.option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX, "2022/1/2[4-8]")
.option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX_MODE, "METADATA_ONLY")
.option(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER, classOf[SparkParquetBootstrapDataProvider].getName)
.mode(SaveMode.Overwrite)
.save(basePath)
val srcPath = "<>/bootstrap-testing/partitioned-parquet-table-date"
val basePath = "<>/bootstrap-testing/bootstrap-hudi-table-2"
val bootstrapDF = spark.emptyDataFrame
bootstrapDF.write
.format("hudi")
.option(HoodieWriteConfig.TABLE_NAME, "hoodie_test")
.option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL)
.option(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY, "key")
.option(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY, "partition")
.option(DataSourceWriteOptions.PRECOMBINE_FIELD_OPT_KEY, "ts")
.option(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH_PROP, srcPath)
.option(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS, classOf[SimpleKeyGenerator].getName)
.option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR, classOf[BootstrapRegexModeSelector].getName)
.option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX, "2022/1/2[4-8]")
.option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX_MODE, "METADATA_ONLY")
.option(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER, classOf[SparkParquetBootstrapDataProvider].getName)
.mode(SaveMode.Overwrite)
.save(basePath) {code}
> Test COW: Spark datasource metadata-only and full-record bootstrap operation
> ----------------------------------------------------------------------------
>
> Key: HUDI-4655
> URL: https://issues.apache.org/jira/browse/HUDI-4655
> Project: Apache Hudi
> Issue Type: Improvement
> Reporter: Ethan Guo
> Assignee: Ethan Guo
> Priority: Blocker
> Fix For: 0.13.0
>
>
> Testing both Hive-style and date (year/month/day) partitions
> {code:java}
> import org.apache.hudi.bootstrap.SparkParquetBootstrapDataProvider
> import org.apache.hudi.client.bootstrap.selector.BootstrapRegexModeSelector
> import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers}
> import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieWriteConfig}
> import org.apache.hudi.keygen.SimpleKeyGenerator
> import org.apache.spark.sql.SaveMode
>
> val srcPath = "<>/Users/ethan/Work/scripts/bootstrap-testing/partitioned-parquet-table"
> val basePath = "<>/Users/ethan/Work/scripts/bootstrap-testing/bootstrap-hudi-table-1"
> val bootstrapDF = spark.emptyDataFrame
> bootstrapDF.write
> .format("hudi")
> .option(HoodieWriteConfig.TABLE_NAME, "hoodie_test")
> .option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL)
> .option(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY, "key")
> .option(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY, "partition")
> .option(DataSourceWriteOptions.PRECOMBINE_FIELD_OPT_KEY, "ts")
> .option(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH_PROP, srcPath)
> .option(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS, classOf[SimpleKeyGenerator].getName)
> .option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR, classOf[BootstrapRegexModeSelector].getName)
> .option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX, "2022/1/2[4-8]")
> .option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX_MODE, "METADATA_ONLY")
> .option(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER, classOf[SparkParquetBootstrapDataProvider].getName)
> .mode(SaveMode.Overwrite)
> .save(basePath)
> val srcPath = "<>/bootstrap-testing/partitioned-parquet-table-date"
> val basePath = "<>/bootstrap-testing/bootstrap-hudi-table-2"
> val bootstrapDF = spark.emptyDataFrame
> bootstrapDF.write
> .format("hudi")
> .option(HoodieWriteConfig.TABLE_NAME, "hoodie_test")
> .option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL)
> .option(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY, "key")
> .option(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY, "partition")
> .option(DataSourceWriteOptions.PRECOMBINE_FIELD_OPT_KEY, "ts")
> .option(HoodieBootstrapConfig.BOOTSTRAP_BASE_PATH_PROP, srcPath)
> .option(HoodieBootstrapConfig.BOOTSTRAP_KEYGEN_CLASS, classOf[SimpleKeyGenerator].getName)
> .option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR, classOf[BootstrapRegexModeSelector].getName)
> .option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX, "2022/1/2[4-8]")
> .option(HoodieBootstrapConfig.BOOTSTRAP_MODE_SELECTOR_REGEX_MODE, "METADATA_ONLY")
> .option(HoodieBootstrapConfig.FULL_BOOTSTRAP_INPUT_PROVIDER, classOf[SparkParquetBootstrapDataProvider].getName)
> .mode(SaveMode.Overwrite)
> .save(basePath) {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)