You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by bh...@apache.org on 2020/01/30 00:26:38 UTC
[incubator-hudi] branch master updated: [HUDI-578] Trim
recordKeyFields and partitionPathFields in ComplexKeyGenerator (#1281)
This is an automated email from the ASF dual-hosted git repository.
bhavanisudha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 652224e [HUDI-578] Trim recordKeyFields and partitionPathFields in ComplexKeyGenerator (#1281)
652224e is described below
commit 652224edc882c083ac46cff095324975e2457004
Author: leesf <49...@qq.com>
AuthorDate: Thu Jan 30 08:26:26 2020 +0800
[HUDI-578] Trim recordKeyFields and partitionPathFields in ComplexKeyGenerator (#1281)
* [HUDI-578] Trim recordKeyFields and partitionPathFields in ComplexKeyGenerator
* add tests
---
.../main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java | 7 +++++--
hudi-spark/src/test/scala/TestDataSourceDefaults.scala | 11 +++++++++++
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
index 7406b9c..80d5488 100644
--- a/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
+++ b/hudi-spark/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java
@@ -28,6 +28,7 @@ import org.apache.avro.generic.GenericRecord;
import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
/**
* Complex key generator, which takes names of fields to be used for recordKey and partitionPath as configs.
@@ -48,9 +49,11 @@ public class ComplexKeyGenerator extends KeyGenerator {
public ComplexKeyGenerator(TypedProperties props) {
super(props);
- this.recordKeyFields = Arrays.asList(props.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(","));
+ this.recordKeyFields = Arrays.asList(props.getString(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY()).split(","))
+ .stream().map(String::trim).collect(Collectors.toList());
this.partitionPathFields =
- Arrays.asList(props.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY()).split(","));
+ Arrays.asList(props.getString(DataSourceWriteOptions.PARTITIONPATH_FIELD_OPT_KEY()).split(","))
+ .stream().map(String::trim).collect(Collectors.toList());
this.hiveStylePartitioning = props.getBoolean(DataSourceWriteOptions.HIVE_STYLE_PARTITIONING_OPT_KEY(),
Boolean.parseBoolean(DataSourceWriteOptions.DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL()));
}
diff --git a/hudi-spark/src/test/scala/TestDataSourceDefaults.scala b/hudi-spark/src/test/scala/TestDataSourceDefaults.scala
index 2ac5fce..31a04d6 100644
--- a/hudi-spark/src/test/scala/TestDataSourceDefaults.scala
+++ b/hudi-spark/src/test/scala/TestDataSourceDefaults.scala
@@ -224,6 +224,17 @@ class TestDataSourceDefaults extends AssertionsForJUnit {
case e: HoodieKeyException =>
// do nothing
}
+
+ // reset name and field1 values.
+ baseRecord.put("name", "name1")
+ baseRecord.put("field1", "field1")
+ val hk7 = new ComplexKeyGenerator(getKeyConfig("field1, name", "field1, name", "false")).getKey(baseRecord)
+ assertEquals("field1:field1,name:name1", hk7.getRecordKey)
+ assertEquals("field1/name1", hk7.getPartitionPath)
+
+ val hk8 = new ComplexKeyGenerator(getKeyConfig("field1,", "field1,", "false")).getKey(baseRecord)
+ assertEquals("field1:field1", hk8.getRecordKey)
+ assertEquals("field1", hk8.getPartitionPath)
}
@Test def testGlobalDeleteKeyGenerator() = {