You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/05/09 00:17:10 UTC
[incubator-pinot] branch empty_array_input created (now d5394f3)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch empty_array_input
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at d5394f3 In DataTypeTransformer, support filling in default null value for empty array input
This branch includes the following new commits:
new d5394f3 In DataTypeTransformer, support filling in default null value for empty array input
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: In DataTypeTransformer,
support filling in default null value for empty array input
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch empty_array_input
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit d5394f3789d89084048740271f9620d23fc6a15f
Author: Jackie (Xiaotian) Jiang <xa...@linkedin.com>
AuthorDate: Wed May 8 17:13:40 2019 -0700
In DataTypeTransformer, support filling in default null value for empty array input
In normal case, RecordReader should fill in default null value for null/empty array input.
In case where RecordReader is not implemented to handle default null value, handle it in DataTypeTransformer.
---
.../org/apache/pinot/common/data/FieldSpec.java | 26 +++++++++++-----------
.../recordtransformer/DataTypeTransformer.java | 7 +++---
.../recordtransformer/RecordTransformerTest.java | 18 +++++++++++++++
3 files changed, 34 insertions(+), 17 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java b/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java
index 30be748..00192a6 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java
@@ -49,19 +49,19 @@ public abstract class FieldSpec implements Comparable<FieldSpec>, ConfigNodeLife
// TODO: revisit to see if we allow 0-length byte array
private static final byte[] NULL_BYTE_ARRAY_VALUE = new byte[0];
- private static final Integer DEFAULT_DIMENSION_NULL_VALUE_OF_INT = Integer.MIN_VALUE;
- private static final Long DEFAULT_DIMENSION_NULL_VALUE_OF_LONG = Long.MIN_VALUE;
- private static final Float DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT = Float.NEGATIVE_INFINITY;
- private static final Double DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE = Double.NEGATIVE_INFINITY;
- private static final byte[] DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
-
- private static final String DEFAULT_DIMENSION_NULL_VALUE_OF_STRING = "null";
- private static final Integer DEFAULT_METRIC_NULL_VALUE_OF_INT = 0;
- private static final Long DEFAULT_METRIC_NULL_VALUE_OF_LONG = 0L;
- private static final Float DEFAULT_METRIC_NULL_VALUE_OF_FLOAT = 0.0F;
- private static final Double DEFAULT_METRIC_NULL_VALUE_OF_DOUBLE = 0.0D;
- private static final String DEFAULT_METRIC_NULL_VALUE_OF_STRING = "null";
- private static final byte[] DEFAULT_METRIC_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
+ public static final Integer DEFAULT_DIMENSION_NULL_VALUE_OF_INT = Integer.MIN_VALUE;
+ public static final Long DEFAULT_DIMENSION_NULL_VALUE_OF_LONG = Long.MIN_VALUE;
+ public static final Float DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT = Float.NEGATIVE_INFINITY;
+ public static final Double DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE = Double.NEGATIVE_INFINITY;
+ public static final String DEFAULT_DIMENSION_NULL_VALUE_OF_STRING = "null";
+ public static final byte[] DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
+
+ public static final Integer DEFAULT_METRIC_NULL_VALUE_OF_INT = 0;
+ public static final Long DEFAULT_METRIC_NULL_VALUE_OF_LONG = 0L;
+ public static final Float DEFAULT_METRIC_NULL_VALUE_OF_FLOAT = 0.0F;
+ public static final Double DEFAULT_METRIC_NULL_VALUE_OF_DOUBLE = 0.0D;
+ public static final String DEFAULT_METRIC_NULL_VALUE_OF_STRING = "null";
+ public static final byte[] DEFAULT_METRIC_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
@ConfigKey("name")
protected String _name;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
index a021f4b..4fff5fc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
@@ -74,9 +74,9 @@ public class DataTypeTransformer implements RecordTransformer {
PinotDataType dest = entry.getValue();
Object value = record.getValue(column);
- // NOTE: should not be null in normal case if combined with other transformers (without TimeTransformer, outgoing
- // time column might be null)
- if (value == null) {
+ // NOTE: should not need to set default null value in normal case (RecordReader is responsible for filling in the
+ // default null value; TimeTransformer is responsible for filling in the outgoing time value if not exists)
+ if (value == null || (value instanceof Object[] && ((Object[]) value).length == 0)) {
// Set default null value
FieldSpec fieldSpec = _schema.getFieldSpecFor(column);
Object defaultNullValue = fieldSpec.getDefaultNullValue();
@@ -91,7 +91,6 @@ public class DataTypeTransformer implements RecordTransformer {
if (value instanceof Object[]) {
// Multi-valued column
Object[] values = (Object[]) value;
- assert values.length > 0;
source = MULTI_VALUE_TYPE_MAP.get(values[0].getClass());
if (source == null) {
source = PinotDataType.OBJECT_ARRAY;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java
index 73b4476..81283cf 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java
@@ -88,6 +88,24 @@ public class RecordTransformerTest {
assertEquals(record.getValue("svStringWithNullCharacters"), "1\0002\0003");
assertEquals(record.getValue("svStringWithLengthLimit"), "123");
}
+
+ // Test empty record
+ record = new GenericRow();
+ for (int i = 0; i < NUM_ROUNDS; i++) {
+ record = transformer.transform(record);
+ assertNotNull(record);
+ assertEquals(record.getValue("svInt"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT);
+ assertEquals(record.getValue("svLong"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
+ assertEquals(record.getValue("svFloat"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT);
+ assertEquals(record.getValue("svDouble"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE);
+ assertEquals(record.getValue("svBytes"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES);
+ assertEquals(record.getValue("mvInt"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT});
+ assertEquals(record.getValue("mvLong"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG});
+ assertEquals(record.getValue("mvFloat"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT});
+ assertEquals(record.getValue("mvDouble"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE});
+ assertEquals(record.getValue("svStringWithNullCharacters"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING);
+ assertEquals(record.getValue("svStringWithLengthLimit"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING);
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org