You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/05/09 01:49:44 UTC
[incubator-pinot] branch master updated: In DataTypeTransformer,
support filling in default null value for empty array input (#4196)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 578f42e In DataTypeTransformer, support filling in default null value for empty array input (#4196)
578f42e is described below
commit 578f42e93631da10ff5e5152a2a645991319b625
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Wed May 8 18:49:37 2019 -0700
In DataTypeTransformer, support filling in default null value for empty array input (#4196)
In normal case, RecordReader should fill in default null value for null/empty array input.
In case where RecordReader is not implemented to handle default null value, handle it in DataTypeTransformer.
---
.../org/apache/pinot/common/data/FieldSpec.java | 26 +++++++++++-----------
.../recordtransformer/DataTypeTransformer.java | 7 +++---
.../recordtransformer/RecordTransformerTest.java | 18 +++++++++++++++
3 files changed, 34 insertions(+), 17 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java b/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java
index 30be748..00192a6 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java
@@ -49,19 +49,19 @@ public abstract class FieldSpec implements Comparable<FieldSpec>, ConfigNodeLife
// TODO: revisit to see if we allow 0-length byte array
private static final byte[] NULL_BYTE_ARRAY_VALUE = new byte[0];
- private static final Integer DEFAULT_DIMENSION_NULL_VALUE_OF_INT = Integer.MIN_VALUE;
- private static final Long DEFAULT_DIMENSION_NULL_VALUE_OF_LONG = Long.MIN_VALUE;
- private static final Float DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT = Float.NEGATIVE_INFINITY;
- private static final Double DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE = Double.NEGATIVE_INFINITY;
- private static final byte[] DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
-
- private static final String DEFAULT_DIMENSION_NULL_VALUE_OF_STRING = "null";
- private static final Integer DEFAULT_METRIC_NULL_VALUE_OF_INT = 0;
- private static final Long DEFAULT_METRIC_NULL_VALUE_OF_LONG = 0L;
- private static final Float DEFAULT_METRIC_NULL_VALUE_OF_FLOAT = 0.0F;
- private static final Double DEFAULT_METRIC_NULL_VALUE_OF_DOUBLE = 0.0D;
- private static final String DEFAULT_METRIC_NULL_VALUE_OF_STRING = "null";
- private static final byte[] DEFAULT_METRIC_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
+ public static final Integer DEFAULT_DIMENSION_NULL_VALUE_OF_INT = Integer.MIN_VALUE;
+ public static final Long DEFAULT_DIMENSION_NULL_VALUE_OF_LONG = Long.MIN_VALUE;
+ public static final Float DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT = Float.NEGATIVE_INFINITY;
+ public static final Double DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE = Double.NEGATIVE_INFINITY;
+ public static final String DEFAULT_DIMENSION_NULL_VALUE_OF_STRING = "null";
+ public static final byte[] DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
+
+ public static final Integer DEFAULT_METRIC_NULL_VALUE_OF_INT = 0;
+ public static final Long DEFAULT_METRIC_NULL_VALUE_OF_LONG = 0L;
+ public static final Float DEFAULT_METRIC_NULL_VALUE_OF_FLOAT = 0.0F;
+ public static final Double DEFAULT_METRIC_NULL_VALUE_OF_DOUBLE = 0.0D;
+ public static final String DEFAULT_METRIC_NULL_VALUE_OF_STRING = "null";
+ public static final byte[] DEFAULT_METRIC_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE;
@ConfigKey("name")
protected String _name;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
index a021f4b..4fff5fc 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java
@@ -74,9 +74,9 @@ public class DataTypeTransformer implements RecordTransformer {
PinotDataType dest = entry.getValue();
Object value = record.getValue(column);
- // NOTE: should not be null in normal case if combined with other transformers (without TimeTransformer, outgoing
- // time column might be null)
- if (value == null) {
+ // NOTE: should not need to set default null value in normal case (RecordReader is responsible for filling in the
+ // default null value; TimeTransformer is responsible for filling in the outgoing time value if not exists)
+ if (value == null || (value instanceof Object[] && ((Object[]) value).length == 0)) {
// Set default null value
FieldSpec fieldSpec = _schema.getFieldSpecFor(column);
Object defaultNullValue = fieldSpec.getDefaultNullValue();
@@ -91,7 +91,6 @@ public class DataTypeTransformer implements RecordTransformer {
if (value instanceof Object[]) {
// Multi-valued column
Object[] values = (Object[]) value;
- assert values.length > 0;
source = MULTI_VALUE_TYPE_MAP.get(values[0].getClass());
if (source == null) {
source = PinotDataType.OBJECT_ARRAY;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java
index 73b4476..81283cf 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java
@@ -88,6 +88,24 @@ public class RecordTransformerTest {
assertEquals(record.getValue("svStringWithNullCharacters"), "1\0002\0003");
assertEquals(record.getValue("svStringWithLengthLimit"), "123");
}
+
+ // Test empty record
+ record = new GenericRow();
+ for (int i = 0; i < NUM_ROUNDS; i++) {
+ record = transformer.transform(record);
+ assertNotNull(record);
+ assertEquals(record.getValue("svInt"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT);
+ assertEquals(record.getValue("svLong"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG);
+ assertEquals(record.getValue("svFloat"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT);
+ assertEquals(record.getValue("svDouble"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE);
+ assertEquals(record.getValue("svBytes"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES);
+ assertEquals(record.getValue("mvInt"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT});
+ assertEquals(record.getValue("mvLong"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG});
+ assertEquals(record.getValue("mvFloat"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT});
+ assertEquals(record.getValue("mvDouble"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE});
+ assertEquals(record.getValue("svStringWithNullCharacters"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING);
+ assertEquals(record.getValue("svStringWithLengthLimit"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING);
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org