You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2023/05/17 00:32:16 UTC
[pinot] branch master updated: [feature] [null support # 9] Support null in time related transform functions. (#10598)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 12d86902a8 [feature] [null support # 9] Support null in time related transform functions. (#10598)
12d86902a8 is described below
commit 12d86902a84d4bc78b6f2f7bc8bd002659ee61cb
Author: Yao Liu <ya...@startree.ai>
AuthorDate: Tue May 16 17:32:09 2023 -0700
[feature] [null support # 9] Support null in time related transform functions. (#10598)
---
.../DateTimeConversionTransformFunction.java | 14 ++++++---
.../function/DateTimeTransformFunction.java | 6 ++++
.../function/DateTruncTransformFunction.java | 6 ++++
.../function/ExtractTransformFunction.java | 6 ++++
.../function/TimeConversionTransformFunction.java | 6 ++++
.../function/BaseTransformFunctionTest.java | 8 +++++
.../DateTimeConversionTransformFunctionTest.java | 24 +++++++++++++++
.../function/DateTimeTransformFunctionTest.java | 21 +++++++++++++
.../function/ExtractTransformFunctionTest.java | 24 ++++++++++++++-
.../TimeConversionTransformFunctionTest.java | 36 ++++++++++++++++++++++
10 files changed, 145 insertions(+), 6 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java
index d2e5308716..c5e255bf25 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java
@@ -30,6 +30,7 @@ import org.apache.pinot.core.operator.transform.transformer.datetime.EpochToSDFT
import org.apache.pinot.core.operator.transform.transformer.datetime.SDFToEpochTransformer;
import org.apache.pinot.core.operator.transform.transformer.datetime.SDFToSDFTransformer;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
+import org.roaringbitmap.RoaringBitmap;
/**
@@ -95,7 +96,6 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction {
if (arguments.size() != 4) {
throw new IllegalArgumentException("Exactly 4 arguments are required for DATE_TIME_CONVERT transform function");
}
-
TransformFunction firstArgument = arguments.get(0);
if (firstArgument instanceof LiteralTransformFunction || !firstArgument.getResultMetadata().isSingleValue()) {
throw new IllegalArgumentException(
@@ -106,8 +106,8 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction {
_dateTimeTransformer = DateTimeTransformerFactory.getDateTimeTransformer(
((LiteralTransformFunction) arguments.get(1)).getStringLiteral(),
- ((LiteralTransformFunction) arguments.get(2)).getStringLiteral(),
- ((LiteralTransformFunction) arguments.get(3)).getStringLiteral());
+ ((LiteralTransformFunction) arguments.get(2)).getStringLiteral(),
+ ((LiteralTransformFunction) arguments.get(3)).getStringLiteral());
if (_dateTimeTransformer instanceof EpochToEpochTransformer
|| _dateTimeTransformer instanceof SDFToEpochTransformer) {
_resultMetadata = LONG_SV_NO_DICTIONARY_METADATA;
@@ -130,8 +130,7 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction {
initLongValuesSV(length);
if (_dateTimeTransformer instanceof EpochToEpochTransformer) {
EpochToEpochTransformer dateTimeTransformer = (EpochToEpochTransformer) _dateTimeTransformer;
- dateTimeTransformer.transform(_mainTransformFunction.transformToLongValuesSV(valueBlock), _longValuesSV,
- length);
+ dateTimeTransformer.transform(_mainTransformFunction.transformToLongValuesSV(valueBlock), _longValuesSV, length);
} else {
SDFToEpochTransformer dateTimeTransformer = (SDFToEpochTransformer) _dateTimeTransformer;
dateTimeTransformer.transform(_mainTransformFunction.transformToStringValuesSV(valueBlock), _longValuesSV,
@@ -158,4 +157,9 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction {
}
return _stringValuesSV;
}
+
+ @Override
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ return _mainTransformFunction.getNullBitmap(valueBlock);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunction.java
index 0f72c39ff4..2e6fe01657 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunction.java
@@ -30,6 +30,7 @@ import org.joda.time.Chronology;
import org.joda.time.DateTimeField;
import org.joda.time.DateTimeZone;
import org.joda.time.chrono.ISOChronology;
+import org.roaringbitmap.RoaringBitmap;
public abstract class DateTimeTransformFunction extends BaseTransformFunction {
@@ -259,4 +260,9 @@ public abstract class DateTimeTransformFunction extends BaseTransformFunction {
}
}
}
+
+ @Override
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ return _timestampsFunction.getNullBitmap(valueBlock);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java
index 83d488038e..f10d0bfa43 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java
@@ -28,6 +28,7 @@ import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ValueBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.joda.time.DateTimeField;
+import org.roaringbitmap.RoaringBitmap;
/**
@@ -138,4 +139,9 @@ public class DateTruncTransformFunction extends BaseTransformFunction {
}
return _longValuesSV;
}
+
+ @Override
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ return _mainTransformFunction.getNullBitmap(valueBlock);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunction.java
index 9ea960b6e1..b92d19c270 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunction.java
@@ -26,6 +26,7 @@ import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.joda.time.Chronology;
import org.joda.time.DateTimeField;
import org.joda.time.chrono.ISOChronology;
+import org.roaringbitmap.RoaringBitmap;
public class ExtractTransformFunction extends BaseTransformFunction {
@@ -100,4 +101,9 @@ public class ExtractTransformFunction extends BaseTransformFunction {
}
}
}
+
+ @Override
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ return _mainTransformFunction.getNullBitmap(valueBlock);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java
index 9e238c6cff..bac3c9bc62 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java
@@ -26,6 +26,7 @@ import org.apache.pinot.core.operator.blocks.ValueBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.core.operator.transform.transformer.timeunit.TimeUnitTransformer;
import org.apache.pinot.core.operator.transform.transformer.timeunit.TimeUnitTransformerFactory;
+import org.roaringbitmap.RoaringBitmap;
public class TimeConversionTransformFunction extends BaseTransformFunction {
@@ -71,4 +72,9 @@ public class TimeConversionTransformFunction extends BaseTransformFunction {
_timeUnitTransformer.transform(_mainTransformFunction.transformToLongValuesSV(valueBlock), _longValuesSV, length);
return _longValuesSV;
}
+
+ @Override
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ return _mainTransformFunction.getNullBitmap(valueBlock);
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
index 641d6a6422..9d79f77cec 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
@@ -98,6 +98,8 @@ public abstract class BaseTransformFunctionTest {
protected static final String STRING_LONG_MV_COLUMN = "stringLongMV";
protected static final String TIME_COLUMN = "timeColumn";
protected static final String TIMESTAMP_COLUMN = "timestampColumn";
+ protected static final String TIMESTAMP_COLUMN_NULL = "timestampColumnNull";
+
protected static final String JSON_COLUMN = "json";
protected static final String DEFAULT_JSON_COLUMN = "defaultJson";
protected final int[] _intSVValues = new int[NUM_ROWS];
@@ -200,6 +202,11 @@ public abstract class BaseTransformFunctionTest {
map.put(STRING_ALPHANUM_MV_COLUMN, _stringAlphaNumericMVValues[i]);
map.put(STRING_LONG_MV_COLUMN, _stringLongFormatMVValues[i]);
map.put(TIMESTAMP_COLUMN, _timeValues[i]);
+ if (i % 2 == 0) {
+ map.put(TIMESTAMP_COLUMN_NULL, _timeValues[i]);
+ } else {
+ map.put(TIMESTAMP_COLUMN_NULL, null);
+ }
map.put(TIME_COLUMN, _timeValues[i]);
_jsonValues[i] = JsonUtils.objectToJsonNode(map).toString();
map.put(JSON_COLUMN, _jsonValues[i]);
@@ -230,6 +237,7 @@ public abstract class BaseTransformFunctionTest {
.addMultiValueDimension(STRING_ALPHANUM_MV_COLUMN, FieldSpec.DataType.STRING)
.addMultiValueDimension(STRING_LONG_MV_COLUMN, FieldSpec.DataType.STRING)
.addDateTime(TIMESTAMP_COLUMN, FieldSpec.DataType.TIMESTAMP, "1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
+ .addDateTime(TIMESTAMP_COLUMN_NULL, FieldSpec.DataType.TIMESTAMP, "1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
.addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, TIME_COLUMN), null).build();
TableConfig tableConfig =
new TableConfigBuilder(TableType.OFFLINE).setTableName("test").setTimeColumnName(TIME_COLUMN)
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java
index 2e1375f5e5..c4cae05ef1 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunctionTest.java
@@ -24,6 +24,7 @@ import org.apache.pinot.common.request.context.RequestContextUtils;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.exception.BadQueryRequestException;
+import org.roaringbitmap.RoaringBitmap;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -71,4 +72,27 @@ public class DateTimeConversionTransformFunctionTest extends BaseTransformFuncti
}
};
}
+
+ @Test
+ public void testDateTimeConversionTransformFunctionNullColumn() {
+ ExpressionContext expression = RequestContextUtils.getExpression(
+ String.format("dateTimeConvert(%s,'1:MILLISECONDS:EPOCH','1:MINUTES:EPOCH','1:MINUTES')",
+ TIMESTAMP_COLUMN_NULL));
+ TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
+ assertTrue(transformFunction instanceof DateTimeConversionTransformFunction);
+ assertEquals(transformFunction.getName(), DateTimeConversionTransformFunction.FUNCTION_NAME);
+ TransformResultMetadata resultMetadata = transformFunction.getResultMetadata();
+ assertTrue(resultMetadata.isSingleValue());
+ assertEquals(resultMetadata.getDataType(), DataType.LONG);
+ long[] expectedValues = new long[NUM_ROWS];
+ RoaringBitmap expectedNulls = new RoaringBitmap();
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (i % 2 == 0) {
+ expectedValues[i] = TimeUnit.MILLISECONDS.toMinutes(_timeValues[i]);
+ } else {
+ expectedNulls.add(i);
+ }
+ }
+ testTransformFunctionWithNull(transformFunction, expectedValues, expectedNulls);
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java
index 7918f269fc..31f786ca3c 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/DateTimeTransformFunctionTest.java
@@ -19,14 +19,17 @@
package org.apache.pinot.core.operator.transform.function;
import java.util.function.LongToIntFunction;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.scalar.DateTimeFunctions;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
+import org.roaringbitmap.RoaringBitmap;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
public class DateTimeTransformFunctionTest extends BaseTransformFunctionTest {
@@ -89,6 +92,24 @@ public class DateTimeTransformFunctionTest extends BaseTransformFunctionTest {
}
}
+ @Test(dataProvider = "testCasesUTC")
+ public void testUTCNullColumn(String function, LongToIntFunction expected,
+ Class<? extends TransformFunction> expectedClass) {
+ ExpressionContext expression =
+ RequestContextUtils.getExpression(String.format("%s(%s)", function, TIMESTAMP_COLUMN_NULL));
+ TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
+ Assert.assertTrue(expectedClass.isInstance(transformFunction));
+ Pair<int[], RoaringBitmap> values = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+
+ for (int i = 0; i < _projectionBlock.getNumDocs(); i++) {
+ if (i % 2 == 0) {
+ assertEquals(values.getLeft()[i], expected.applyAsInt(_timeValues[i]));
+ } else {
+ assertTrue(values.getRight().contains(i));
+ }
+ }
+ }
+
@Test(dataProvider = "testCasesZoned")
public void testZoned(String function, ZonedTimeFunction expected, Class<? extends TransformFunction> expectedClass) {
for (String zone : new String[]{"Europe/Berlin", "America/New_York", "Asia/Katmandu"}) {
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java
index 1c78b0def8..a821d22f98 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ExtractTransformFunctionTest.java
@@ -19,14 +19,17 @@
package org.apache.pinot.core.operator.transform.function;
import java.util.function.LongToIntFunction;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.function.scalar.DateTimeFunctions;
import org.apache.pinot.common.request.context.ExpressionContext;
import org.apache.pinot.common.request.context.RequestContextUtils;
+import org.roaringbitmap.RoaringBitmap;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
public class ExtractTransformFunctionTest extends BaseTransformFunctionTest {
@@ -44,7 +47,7 @@ public class ExtractTransformFunctionTest extends BaseTransformFunctionTest {
// TODO: Need to add timezone_hour and timezone_minute
// "timezone_hour",
// "timezone_minute",
- //@formatter:on
+ //@formatter:on
};
}
@@ -63,4 +66,23 @@ public class ExtractTransformFunctionTest extends BaseTransformFunctionTest {
assertEquals(value[i], expected.applyAsInt(_timeValues[i]));
}
}
+
+ @Test(dataProvider = "testCases")
+ public void testExtractTransformFunctionNull(String field, LongToIntFunction expected) {
+ // NOTE: functionality of ExtractTransformFunction is covered in ExtractTransformFunctionTest
+ // SELECT EXTRACT(YEAR FROM '2017-10-10')
+ ExpressionContext expression =
+ RequestContextUtils.getExpression(String.format("extract(%s FROM %s)", field, TIMESTAMP_COLUMN_NULL));
+
+ TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
+ Assert.assertTrue(transformFunction instanceof ExtractTransformFunction);
+ Pair<int[], RoaringBitmap> valuesSVWithNull = transformFunction.transformToIntValuesSVWithNull(_projectionBlock);
+ for (int i = 0; i < _projectionBlock.getNumDocs(); i++) {
+ if (i % 2 == 0) {
+ assertEquals(valuesSVWithNull.getLeft()[i], expected.applyAsInt(_timeValues[i]));
+ } else {
+ assertTrue(valuesSVWithNull.getRight().contains(i));
+ }
+ }
+ }
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java
index 2540f6c9cf..e01ccbcbb5 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunctionTest.java
@@ -24,6 +24,7 @@ import org.apache.pinot.common.request.context.RequestContextUtils;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.exception.BadQueryRequestException;
+import org.roaringbitmap.RoaringBitmap;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@@ -51,6 +52,28 @@ public class TimeConversionTransformFunctionTest extends BaseTransformFunctionTe
testTransformFunction(transformFunction, expectedValues);
}
+ @Test(dataProvider = "testTimeConversionTransformFunctionNull")
+ public void testTimeConversionTransformFunctionNullColumn(String expressionStr) {
+ ExpressionContext expression = RequestContextUtils.getExpression(expressionStr);
+ TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
+ assertTrue(transformFunction instanceof TimeConversionTransformFunction);
+ assertEquals(transformFunction.getName(), TimeConversionTransformFunction.FUNCTION_NAME);
+ TransformResultMetadata resultMetadata = transformFunction.getResultMetadata();
+ assertEquals(resultMetadata.getDataType(), DataType.LONG);
+ assertTrue(resultMetadata.isSingleValue());
+ assertFalse(resultMetadata.hasDictionary());
+ long[] expectedValues = new long[NUM_ROWS];
+ RoaringBitmap expectedNull = new RoaringBitmap();
+ for (int i = 0; i < NUM_ROWS; i++) {
+ if (i % 2 == 0) {
+ expectedValues[i] = TimeUnit.MILLISECONDS.toDays(_timeValues[i]);
+ } else {
+ expectedNull.add(i);
+ }
+ }
+ testTransformFunctionWithNull(transformFunction, expectedValues, expectedNull);
+ }
+
@DataProvider(name = "testTimeConversionTransformFunction")
public Object[][] testTimeConversionTransformFunction() {
return new Object[][]{
@@ -64,6 +87,19 @@ public class TimeConversionTransformFunctionTest extends BaseTransformFunctionTe
};
}
+ @DataProvider(name = "testTimeConversionTransformFunctionNull")
+ public Object[][] testTimeConversionTransformFunctionNull() {
+ return new Object[][]{
+ new Object[]{
+ String.format("timeConvert(%s,'MILLISECONDS','DAYS')", TIMESTAMP_COLUMN_NULL)
+ }, new Object[]{
+ String.format(
+ "timeConvert(timeConvert(timeConvert(%s,'MILLISECONDS','SECONDS'),'SECONDS','HOURS'),'HOURS','DAYS')",
+ TIMESTAMP_COLUMN_NULL)
+ }
+ };
+ }
+
@Test(dataProvider = "testIllegalArguments", expectedExceptions = {BadQueryRequestException.class})
public void testIllegalArguments(String expressionStr) {
ExpressionContext expression = RequestContextUtils.getExpression(expressionStr);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org