You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by ud...@apache.org on 2021/08/20 09:44:55 UTC
[hudi] branch release-0.9.0 updated: Keep non-conflicting names for
common configs between DataSourceOptions and HoodieWriteConfig (#3511)
This is an automated email from the ASF dual-hosted git repository.
uditme pushed a commit to branch release-0.9.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/release-0.9.0 by this push:
new 2e69c23 Keep non-conflicting names for common configs between DataSourceOptions and HoodieWriteConfig (#3511)
2e69c23 is described below
commit 2e69c23a29de350dcc9b92a3b1cb12c133eca036
Author: Udit Mehrotra <ud...@gmail.com>
AuthorDate: Fri Aug 20 02:42:59 2021 -0700
Keep non-conflicting names for common configs between DataSourceOptions and HoodieWriteConfig (#3511)
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 10 ++++-----
.../hudi/keygen/ComplexAvroKeyGenerator.java | 4 ++--
.../apache/hudi/keygen/CustomAvroKeyGenerator.java | 4 ++--
.../hudi/keygen/GlobalAvroDeleteKeyGenerator.java | 2 +-
.../keygen/NonpartitionedAvroKeyGenerator.java | 2 +-
.../apache/hudi/keygen/SimpleAvroKeyGenerator.java | 4 ++--
.../keygen/TimestampBasedAvroKeyGenerator.java | 4 ++--
.../table/upgrade/BaseOneToTwoUpgradeHandler.java | 2 +-
...estCreateAvroKeyGeneratorByTypeWithFactory.java | 6 ++---
.../factory/TestHoodieAvroKeyGeneratorFactory.java | 6 ++---
.../hudi/table/upgrade/OneToTwoUpgradeHandler.java | 2 +-
.../apache/hudi/keygen/ComplexKeyGenerator.java | 4 ++--
.../org/apache/hudi/keygen/CustomKeyGenerator.java | 4 ++--
.../hudi/keygen/GlobalDeleteKeyGenerator.java | 2 +-
.../hudi/keygen/NonpartitionedKeyGenerator.java | 2 +-
.../org/apache/hudi/keygen/SimpleKeyGenerator.java | 4 ++--
.../hudi/keygen/TimestampBasedKeyGenerator.java | 4 ++--
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 2 +-
.../hudi/keygen/TestComplexKeyGenerator.java | 26 +++++++++++-----------
.../apache/hudi/keygen/TestCustomKeyGenerator.java | 22 +++++++++---------
.../hudi/keygen/TestGlobalDeleteKeyGenerator.java | 12 +++++-----
.../keygen/TestNonpartitionedKeyGenerator.java | 22 +++++++++---------
.../apache/hudi/keygen/TestSimpleKeyGenerator.java | 22 +++++++++---------
.../keygen/TestTimestampBasedKeyGenerator.java | 6 ++---
.../TestCreateKeyGeneratorByTypeWithFactory.java | 6 ++---
.../TestHoodieSparkKeyGeneratorFactory.java | 6 ++---
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 12 +++++-----
.../org/apache/hudi/keygen/BaseKeyGenerator.java | 4 ++--
.../hudi/keygen/constant/KeyGeneratorOptions.java | 22 +++++++++---------
.../apache/hudi/configuration/FlinkOptions.java | 6 ++---
.../scala/org/apache/hudi/DataSourceOptions.scala | 18 +++++++--------
31 files changed, 126 insertions(+), 126 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 448ce9f..d16d417 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -84,7 +84,7 @@ public class HoodieWriteConfig extends HoodieConfig {
.noDefaultValue()
.withDocumentation("Table name that will be used for registering with metastores like HMS. Needs to be same across runs.");
- public static final ConfigProperty<String> PRECOMBINE_FIELD = ConfigProperty
+ public static final ConfigProperty<String> PRECOMBINE_FIELD_NAME = ConfigProperty
.key("hoodie.datasource.write.precombine.field")
.defaultValue("ts")
.withDocumentation("Field used in preCombining before actual write. When two records have the same key value, "
@@ -423,10 +423,10 @@ public class HoodieWriteConfig extends HoodieConfig {
@Deprecated
public static final String TABLE_NAME = TBL_NAME.key();
/**
- * @deprecated Use {@link #PRECOMBINE_FIELD} and its methods instead
+ * @deprecated Use {@link #PRECOMBINE_FIELD_NAME} and its methods instead
*/
@Deprecated
- public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD.key();
+ public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD_NAME.key();
/**
* @deprecated Use {@link #WRITE_PAYLOAD_CLASS_NAME} and its methods instead
*/
@@ -859,7 +859,7 @@ public class HoodieWriteConfig extends HoodieConfig {
}
public String getPreCombineField() {
- return getString(PRECOMBINE_FIELD);
+ return getString(PRECOMBINE_FIELD_NAME);
}
public String getWritePayloadClass() {
@@ -1803,7 +1803,7 @@ public class HoodieWriteConfig extends HoodieConfig {
}
public Builder withPreCombineField(String preCombineField) {
- writeConfig.setValue(PRECOMBINE_FIELD, preCombineField);
+ writeConfig.setValue(PRECOMBINE_FIELD_NAME, preCombineField);
return this;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java
index 87ddddd..375225f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java
@@ -32,9 +32,9 @@ public class ComplexAvroKeyGenerator extends BaseKeyGenerator {
public ComplexAvroKeyGenerator(TypedProperties props) {
super(props);
- this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())
+ this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
- this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())
+ this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java
index 286e495..370b57b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java
@@ -55,8 +55,8 @@ public class CustomAvroKeyGenerator extends BaseKeyGenerator {
public CustomAvroKeyGenerator(TypedProperties props) {
super(props);
- this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList());
- this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList());
+ this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
+ this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java
index 3b20c8a..674e286 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java
@@ -35,7 +35,7 @@ public class GlobalAvroDeleteKeyGenerator extends BaseKeyGenerator {
public GlobalAvroDeleteKeyGenerator(TypedProperties config) {
super(config);
- this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(","));
+ this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(","));
}
@Override
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java
index b1e0ca7..4384f17 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java
@@ -36,7 +36,7 @@ public class NonpartitionedAvroKeyGenerator extends BaseKeyGenerator {
public NonpartitionedAvroKeyGenerator(TypedProperties props) {
super(props);
- this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())
+ this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
this.partitionPathFields = EMPTY_PARTITION_FIELD_LIST;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java
index 2aa9392..dae55c5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java
@@ -29,8 +29,8 @@ import java.util.Collections;
public class SimpleAvroKeyGenerator extends BaseKeyGenerator {
public SimpleAvroKeyGenerator(TypedProperties props) {
- this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()),
- props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()));
+ this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
+ props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
}
SimpleAvroKeyGenerator(TypedProperties props, String partitionPathField) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
index 678591e..430cde8 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
@@ -88,8 +88,8 @@ public class TimestampBasedAvroKeyGenerator extends SimpleAvroKeyGenerator {
}
public TimestampBasedAvroKeyGenerator(TypedProperties config) throws IOException {
- this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()),
- config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()));
+ this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
+ config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
}
TimestampBasedAvroKeyGenerator(TypedProperties config, String partitionPathField) throws IOException {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java
index 6a43f61..e3a14e4 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java
@@ -33,7 +33,7 @@ public abstract class BaseOneToTwoUpgradeHandler implements UpgradeHandler {
public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime) {
Map<ConfigProperty, String> tablePropsToAdd = new HashMap<>();
tablePropsToAdd.put(HoodieTableConfig.PARTITION_FIELDS, getPartitionColumns(config));
- tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()));
+ tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()));
tablePropsToAdd.put(HoodieTableConfig.BASE_FILE_FORMAT, config.getString(HoodieTableConfig.BASE_FILE_FORMAT));
return tablePropsToAdd;
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java
index 7983c66..b69d844 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java
@@ -52,9 +52,9 @@ public class TestCreateAvroKeyGeneratorByTypeWithFactory {
@BeforeEach
public void init() {
props = new TypedProperties();
- props.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
- props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
- props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ props.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+ props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
+ props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
// for timestamp based key generator
props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING");
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java
index c17bc1b..c3be628 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java
@@ -67,9 +67,9 @@ public class TestHoodieAvroKeyGeneratorFactory {
private TypedProperties getCommonProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
- properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java
index 3aa9f9b..b84ce6d 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java
@@ -25,6 +25,6 @@ public class OneToTwoUpgradeHandler extends BaseOneToTwoUpgradeHandler {
@Override
String getPartitionColumns(HoodieWriteConfig config) {
- return config.getProps().getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key());
+ return config.getProps().getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key());
}
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
index 6794fc1..8d02ce6 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
@@ -37,9 +37,9 @@ public class ComplexKeyGenerator extends BuiltinKeyGenerator {
public ComplexKeyGenerator(TypedProperties props) {
super(props);
- this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())
+ this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
- this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())
+ this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList());
complexAvroKeyGenerator = new ComplexAvroKeyGenerator(props);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java
index 2ad0ace..c43892a 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java
@@ -53,8 +53,8 @@ public class CustomKeyGenerator extends BuiltinKeyGenerator {
public CustomKeyGenerator(TypedProperties props) {
super(props);
- this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList());
- this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList());
+ this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
+ this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList());
customAvroKeyGenerator = new CustomAvroKeyGenerator(props);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java
index e90a100..391ea2c 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java
@@ -39,7 +39,7 @@ public class GlobalDeleteKeyGenerator extends BuiltinKeyGenerator {
private final GlobalAvroDeleteKeyGenerator globalAvroDeleteKeyGenerator;
public GlobalDeleteKeyGenerator(TypedProperties config) {
super(config);
- this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(","));
+ this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(","));
globalAvroDeleteKeyGenerator = new GlobalAvroDeleteKeyGenerator(config);
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java
index 6a9dbaf..1664c86 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java
@@ -39,7 +39,7 @@ public class NonpartitionedKeyGenerator extends BuiltinKeyGenerator {
public NonpartitionedKeyGenerator(TypedProperties props) {
super(props);
- this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())
+ this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.split(",")).map(String::trim).collect(Collectors.toList());
this.partitionPathFields = Collections.emptyList();
nonpartitionedAvroKeyGenerator = new NonpartitionedAvroKeyGenerator(props);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java
index e82bf75..60765b0 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java
@@ -36,8 +36,8 @@ public class SimpleKeyGenerator extends BuiltinKeyGenerator {
private final SimpleAvroKeyGenerator simpleAvroKeyGenerator;
public SimpleKeyGenerator(TypedProperties props) {
- this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()),
- props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()));
+ this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
+ props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
}
SimpleKeyGenerator(TypedProperties props, String partitionPathField) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java
index b4bca9f..03d1e92 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java
@@ -41,8 +41,8 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
private final TimestampBasedAvroKeyGenerator timestampBasedAvroKeyGenerator;
public TimestampBasedKeyGenerator(TypedProperties config) throws IOException {
- this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()),
- config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()));
+ this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()),
+ config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
}
TimestampBasedKeyGenerator(TypedProperties config, String partitionPathField) throws IOException {
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
index 239e101..ed0ab97 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala
@@ -258,7 +258,7 @@ object HoodieSparkUtils extends SparkAdapterSupport {
.mkString(",")
case b: BaseKeyGenerator => b.getPartitionPathFields.asScala.mkString(",")
- case _ => typedProperties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())
+ case _ => typedProperties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java
index f4f6a6c..87deef2 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java
@@ -39,11 +39,11 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey) {
TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col");
} else {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
}
- properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
+ properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties;
}
@@ -53,20 +53,20 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties;
}
private TypedProperties getProps() {
TypedProperties properties = getCommonProps(true);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms");
return properties;
}
@@ -105,8 +105,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
@Test
public void testSingleValueKeyGenerator() {
TypedProperties properties = new TypedProperties();
- properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
- properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+ properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 1);
assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 1);
@@ -128,8 +128,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
@Test
public void testMultipleValueKeyGenerator() {
TypedProperties properties = new TypedProperties();
- properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,timestamp");
- properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "rider,driver");
+ properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp");
+ properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "rider,driver");
ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2);
assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 2);
@@ -154,8 +154,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities {
@Test
public void testMultipleValueKeyGeneratorNonPartitioned() {
TypedProperties properties = new TypedProperties();
- properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,timestamp");
- properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "");
+ properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp");
+ properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2);
assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 0);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
index 9dbb3a72..4bfc71f 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java
@@ -48,53 +48,53 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey, boolean useKeyGeneratorClassName) {
TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col");
} else {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
}
if (useKeyGeneratorClassName) {
properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName());
} else {
properties.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name());
}
- properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
+ properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties;
}
private TypedProperties getPropertiesForSimpleKeyGen(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple");
return properties;
}
private TypedProperties getImproperPartitionFieldFormatProp(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
private TypedProperties getInvalidPartitionKeyTypeProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:dummy");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:dummy");
return properties;
}
private TypedProperties getComplexRecordKeyWithSimplePartitionProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple");
return properties;
}
private TypedProperties getComplexRecordKeyAndPartitionPathProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple,ts_ms:timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple,ts_ms:timestamp");
populateNecessaryPropsForTimestampBasedKeyGen(properties);
return properties;
}
private TypedProperties getPropsWithoutRecordKeyFieldProps(boolean useKeyGeneratorClassName) {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple");
if (useKeyGeneratorClassName) {
properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName());
} else {
@@ -111,14 +111,14 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesForTimestampBasedKeyGen(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "ts_ms:timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "ts_ms:timestamp");
populateNecessaryPropsForTimestampBasedKeyGen(properties);
return properties;
}
private TypedProperties getPropertiesForNonPartitionedKeyGen(boolean useKeyGeneratorClassName) {
TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
return properties;
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java
index 31457bc..aa9568b 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java
@@ -34,29 +34,29 @@ public class TestGlobalDeleteKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey) {
TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,pii_col");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,pii_col");
} else {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
}
- properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
+ properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties;
}
private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties;
}
private TypedProperties getProps() {
TypedProperties properties = getCommonProps(true);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms");
return properties;
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java
index 99dfbdb..0760de1 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java
@@ -37,11 +37,11 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps(boolean getComplexRecordKey) {
TypedProperties properties = new TypedProperties();
if (getComplexRecordKey) {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col");
} else {
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
}
- properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
+ properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties;
}
@@ -51,19 +51,19 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesWithPartitionPathProp() {
TypedProperties properties = getCommonProps(true);
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms");
return properties;
}
private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties;
}
@@ -78,7 +78,7 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties);
GenericRecord record = getRecord();
Row row = KeyGeneratorTestUtilities.getRow(record);
- Assertions.assertEquals(properties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()), "timestamp,ts_ms");
+ Assertions.assertEquals(properties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()), "timestamp,ts_ms");
Assertions.assertEquals(keyGenerator.getPartitionPath(row), "");
}
@@ -101,8 +101,8 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
@Test
public void testSingleValueKeyGeneratorNonPartitioned() {
TypedProperties properties = new TypedProperties();
- properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "timestamp");
- properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "");
+ properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "timestamp");
+ properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties);
assertEquals(keyGenerator.getRecordKeyFields().size(), 1);
assertEquals(keyGenerator.getPartitionPathFields().size(), 0);
@@ -118,8 +118,8 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities {
@Test
public void testMultipleValueKeyGeneratorNonPartitioned1() {
TypedProperties properties = new TypedProperties();
- properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "timestamp,driver");
- properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "");
+ properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "timestamp,driver");
+ properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "");
NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties);
assertEquals(keyGenerator.getRecordKeyFields().size(), 2);
assertEquals(keyGenerator.getPartitionPathFields().size(), 0);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java
index 5fbbd8d..75dca2e 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java
@@ -39,8 +39,8 @@ import static org.apache.hudi.keygen.KeyGenUtils.DEFAULT_PARTITION_PATH;
public class TestSimpleKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getCommonProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
- properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
return properties;
}
@@ -50,40 +50,40 @@ public class TestSimpleKeyGenerator extends KeyGeneratorTestUtilities {
private TypedProperties getPropertiesWithoutRecordKeyProp() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
private TypedProperties getWrongRecordKeyFieldProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key");
return properties;
}
private TypedProperties getWrongPartitionPathFieldProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "_wrong_partition_path");
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "_wrong_partition_path");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
return properties;
}
private TypedProperties getComplexRecordKeyProp() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,pii_col");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,pii_col");
return properties;
}
private TypedProperties getProps() {
TypedProperties properties = getCommonProps();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
private TypedProperties getPropsWithNestedPartitionPathField() {
TypedProperties properties = getCommonProps();
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "nested_col.prop1");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "nested_col.prop1");
return properties;
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java
index 2a38418..88a54c8 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java
@@ -64,9 +64,9 @@ public class TestTimestampBasedKeyGenerator {
baseRow = genericRecordToRow(baseRecord);
internalRow = KeyGeneratorTestUtilities.getInternalRow(baseRow);
- properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "field1");
- properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "createTime");
- properties.setProperty(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "false");
+ properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "field1");
+ properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "createTime");
+ properties.setProperty(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "false");
}
private TypedProperties getBaseKeyConfig(String timestampType, String dateFormat, String timezone, String scalarType) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java
index 10bc506..ad3edd4 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java
@@ -53,9 +53,9 @@ public class TestCreateKeyGeneratorByTypeWithFactory {
@BeforeEach
public void init() {
props = new TypedProperties();
- props.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
- props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
- props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ props.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+ props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
+ props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
// for timestamp based key generator
props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING");
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java
index 682329a..dffe1ea 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java
@@ -74,9 +74,9 @@ public class TestHoodieSparkKeyGeneratorFactory {
private TypedProperties getCommonProps() {
TypedProperties properties = new TypedProperties();
- properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key");
- properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true");
- properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp");
+ properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key");
+ properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true");
+ properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp");
return properties;
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index 123a33c..792da4e 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -231,8 +231,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
}
private void addNewTableParamsToProps(Map<String, String> params) {
- params.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "uuid");
- params.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "partition_path");
+ params.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "uuid");
+ params.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path");
params.put(HoodieTableConfig.NAME.key(), metaClient.getTableConfig().getTableName());
params.put(BASE_FILE_FORMAT.key(), BASE_FILE_FORMAT.defaultValue().name());
}
@@ -262,8 +262,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
private void assertTableProps(HoodieWriteConfig cfg) {
HoodieTableConfig tableConfig = metaClient.getTableConfig();
Properties originalProps = cfg.getProps();
- assertEquals(tableConfig.getPartitionFieldProp(), originalProps.getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()));
- assertEquals(tableConfig.getRecordKeyFieldProp(), originalProps.getProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key()));
+ assertEquals(tableConfig.getPartitionFieldProp(), originalProps.getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
+ assertEquals(tableConfig.getRecordKeyFieldProp(), originalProps.getProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()));
assertEquals(tableConfig.getTableName(), cfg.getTableName());
assertEquals(tableConfig.getBaseFileFormat().name(), originalProps.getProperty(BASE_FILE_FORMAT.key()));
}
@@ -289,8 +289,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
// set table configs
HoodieTableConfig tableConfig = metaClient.getTableConfig();
tableConfig.setValue(HoodieTableConfig.NAME, cfg.getTableName());
- tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()));
- tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()));
+ tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()));
+ tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()));
tableConfig.setValue(BASE_FILE_FORMAT, cfg.getString(BASE_FILE_FORMAT));
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java b/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java
index 654702e..88092da 100644
--- a/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java
+++ b/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java
@@ -37,8 +37,8 @@ public abstract class BaseKeyGenerator extends KeyGenerator {
super(config);
this.encodePartitionPath = config.getBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.key(),
Boolean.parseBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.defaultValue()));
- this.hiveStylePartitioning = config.getBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(),
- Boolean.parseBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.defaultValue()));
+ this.hiveStylePartitioning = config.getBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(),
+ Boolean.parseBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.defaultValue()));
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java
index ab51b8e..3e369a5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java
+++ b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java
@@ -36,21 +36,21 @@ public class KeyGeneratorOptions extends HoodieConfig {
.defaultValue("false")
.withDocumentation("Should we url encode the partition path value, before creating the folder structure.");
- public static final ConfigProperty<String> HIVE_STYLE_PARTITIONING = ConfigProperty
+ public static final ConfigProperty<String> HIVE_STYLE_PARTITIONING_ENABLE = ConfigProperty
.key("hoodie.datasource.write.hive_style_partitioning")
.defaultValue("false")
.withDocumentation("Flag to indicate whether to use Hive style partitioning.\n"
+ "If set true, the names of partition folders follow <partition_column_name>=<partition_value> format.\n"
+ "By default false (the names of partition folders are only partition values)");
- public static final ConfigProperty<String> RECORDKEY_FIELD = ConfigProperty
+ public static final ConfigProperty<String> RECORDKEY_FIELD_NAME = ConfigProperty
.key("hoodie.datasource.write.recordkey.field")
.defaultValue("uuid")
.withDocumentation("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n"
+ "Actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using\n"
+ "the dot notation eg: `a.b.c`");
- public static final ConfigProperty<String> PARTITIONPATH_FIELD = ConfigProperty
+ public static final ConfigProperty<String> PARTITIONPATH_FIELD_NAME = ConfigProperty
.key("hoodie.datasource.write.partitionpath.field")
.defaultValue("partitionpath")
.withDocumentation("Partition path field. Value to be used at the partitionPath component of HoodieKey. "
@@ -67,24 +67,24 @@ public class KeyGeneratorOptions extends HoodieConfig {
@Deprecated
public static final String DEFAULT_URL_ENCODE_PARTITIONING_OPT_VAL = URL_ENCODE_PARTITIONING.defaultValue();
/**
- * @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods.
+ * @deprecated Use {@link #HIVE_STYLE_PARTITIONING_ENABLE} and its methods.
*/
@Deprecated
- public static final String HIVE_STYLE_PARTITIONING_OPT_KEY = HIVE_STYLE_PARTITIONING.key();
+ public static final String HIVE_STYLE_PARTITIONING_OPT_KEY = HIVE_STYLE_PARTITIONING_ENABLE.key();
/**
- * @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods.
+ * @deprecated Use {@link #HIVE_STYLE_PARTITIONING_ENABLE} and its methods.
*/
@Deprecated
- public static final String DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue();
+ public static final String DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING_ENABLE.defaultValue();
/**
- * @deprecated Use {@link #RECORDKEY_FIELD} and its methods.
+ * @deprecated Use {@link #RECORDKEY_FIELD_NAME} and its methods.
*/
@Deprecated
- public static final String RECORDKEY_FIELD_OPT_KEY = RECORDKEY_FIELD.key();
+ public static final String RECORDKEY_FIELD_OPT_KEY = RECORDKEY_FIELD_NAME.key();
/**
- * @deprecated Use {@link #PARTITIONPATH_FIELD} and its methods.
+ * @deprecated Use {@link #PARTITIONPATH_FIELD_NAME} and its methods.
*/
@Deprecated
- public static final String PARTITIONPATH_FIELD_OPT_KEY = PARTITIONPATH_FIELD.key();
+ public static final String PARTITIONPATH_FIELD_OPT_KEY = PARTITIONPATH_FIELD_NAME.key();
}
diff --git a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
index 72e8fcc..a1c4417 100644
--- a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
+++ b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
@@ -280,7 +280,7 @@ public class FlinkOptions extends HoodieConfig {
+ "By default true (in favor of streaming progressing over data integrity)");
public static final ConfigOption<String> RECORD_KEY_FIELD = ConfigOptions
- .key(KeyGeneratorOptions.RECORDKEY_FIELD.key())
+ .key(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
.stringType()
.defaultValue("uuid")
.withDescription("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n"
@@ -288,7 +288,7 @@ public class FlinkOptions extends HoodieConfig {
+ "the dot notation eg: `a.b.c`");
public static final ConfigOption<String> PARTITION_PATH_FIELD = ConfigOptions
- .key(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())
+ .key(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
.stringType()
.defaultValue("")
.withDescription("Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`.\n"
@@ -301,7 +301,7 @@ public class FlinkOptions extends HoodieConfig {
.withDescription("Whether to encode the partition path url, default false");
public static final ConfigOption<Boolean> HIVE_STYLE_PARTITIONING = ConfigOptions
- .key(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key())
+ .key(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key())
.booleanType()
.defaultValue(false)
.withDescription("Whether to use Hive style partitioning.\n"
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala
index 9881e43..3c7fe7d 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala
@@ -65,7 +65,7 @@ object DataSourceReadOptions {
.noDefaultValue()
.withDocumentation("Comma separated list of file paths to read within a Hudi table.")
- val READ_PRE_COMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD
+ val READ_PRE_COMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD_NAME
val ENABLE_HOODIE_FILE_INDEX: ConfigProperty[Boolean] = ConfigProperty
.key("hoodie.file.index.enable")
@@ -238,7 +238,7 @@ object DataSourceWriteOptions {
* key value, we will pick the one with the largest value for the precombine field,
* determined by Object.compareTo(..)
*/
- val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD
+ val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD_NAME
/**
* Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting.
@@ -252,20 +252,20 @@ object DataSourceWriteOptions {
* the dot notation eg: `a.b.c`
*
*/
- val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD
+ val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD_NAME
/**
* Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`. Actual
* value obtained by invoking .toString()
*/
- val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD
+ val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME
/**
* Flag to indicate whether to use Hive style partitioning.
* If set true, the names of partition folders follow <partition_column_name>=<partition_value> format.
* By default false (the names of partition folders are only partition values)
*/
- val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING
+ val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE
val KEYGENERATOR_CLASS_NAME = ConfigProperty.key("hoodie.datasource.write.keygenerator.class")
.defaultValue(classOf[SimpleKeyGenerator].getName)
@@ -565,7 +565,7 @@ object DataSourceWriteOptions {
val DEFAULT_ENABLE_ROW_WRITER_OPT_VAL = ENABLE_ROW_WRITER.defaultValue()
/** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */
@Deprecated
- val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key()
+ val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key()
/** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */
@Deprecated
val DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue()
@@ -604,13 +604,13 @@ object DataSourceWriteOptions {
/** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */
@Deprecated
- val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD.key()
+ val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()
/** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */
@Deprecated
val DEFAULT_RECORDKEY_FIELD_OPT_VAL = RECORDKEY_FIELD.defaultValue()
/** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */
@Deprecated
- val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD.key()
+ val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()
/** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */
@Deprecated
val DEFAULT_PARTITIONPATH_FIELD_OPT_VAL = PARTITIONPATH_FIELD.defaultValue()
@@ -620,7 +620,7 @@ object DataSourceWriteOptions {
val TABLE_NAME_OPT_KEY = TABLE_NAME.key()
/** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */
@Deprecated
- val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD.key()
+ val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD_NAME.key()
/** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */
@Deprecated
val DEFAULT_PRECOMBINE_FIELD_OPT_VAL = PRECOMBINE_FIELD.defaultValue()