You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2023/06/30 22:41:25 UTC
[pinot] branch master updated: When extracting root-level field name for complex type handling, use the whole delimiter (#11005)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fac02cf4df When extracting root-level field name for complex type handling, use the whole delimiter (#11005)
fac02cf4df is described below
commit fac02cf4df857fbf5f3d09ab2c3eedba0b265ce4
Author: Rekha Seethamraju <re...@gmail.com>
AuthorDate: Fri Jun 30 15:41:19 2023 -0700
When extracting root-level field name for complex type handling, use the whole delimiter (#11005)
---
.../apache/pinot/segment/local/utils/IngestionUtils.java | 2 +-
.../pinot/segment/local/utils/IngestionUtilsTest.java | 16 ++++++++++++++++
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
index 72aadb70cb..1d73c4cfd5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java
@@ -328,7 +328,7 @@ public final class IngestionUtils {
String delimiter = complexTypeConfig.getDelimiter() == null ? ComplexTypeTransformer.DEFAULT_DELIMITER
: complexTypeConfig.getDelimiter();
for (String field : fieldsToRead) {
- result.add(StringUtils.split(field, delimiter)[0]);
+ result.add(StringUtils.splitByWholeSeparator(field, delimiter)[0]);
}
return result;
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
index 9b54600e93..817ce0362a 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/IngestionUtilsTest.java
@@ -233,4 +233,20 @@ public class IngestionUtilsTest {
Assert.assertEquals(fields.size(), 1);
Assert.assertTrue(fields.containsAll(Sets.newHashSet("s1")));
}
+
+ @Test
+ public void testComplexTypeConfig() {
+ IngestionConfig ingestionConfig = new IngestionConfig();
+ ComplexTypeConfig complexTypeConfig = new ComplexTypeConfig(null, "__",
+ ComplexTypeConfig.CollectionNotUnnestedToJson.NON_PRIMITIVE, null);
+ Schema schema = new Schema();
+
+ ingestionConfig.setComplexTypeConfig(complexTypeConfig);
+ schema.addField(new DimensionFieldSpec("a_b__c_d", FieldSpec.DataType.STRING, true));
+ schema.addField(new DimensionFieldSpec("f_d", FieldSpec.DataType.STRING, false));
+ schema.addField(new DimensionFieldSpec("ab__cd", FieldSpec.DataType.STRING, true));
+ Set<String> fields = IngestionUtils.getFieldsForRecordExtractor(ingestionConfig, schema);
+ Assert.assertEquals(fields.size(), 3);
+ Assert.assertTrue(fields.containsAll(Sets.newHashSet("a_b", "f_d", "ab")));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org