You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2025/06/06 10:05:51 UTC
(druid) branch master updated: more lenient array expression casting, fix inconsistencies and simplify json_value (#18078)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new a2d326ff7aa more lenient array expression casting, fix inconsistencies and simplify json_value (#18078)
a2d326ff7aa is described below
commit a2d326ff7aaf5f2acbd6c8ad81a042f06b79c32f
Author: Clint Wylie <cw...@apache.org>
AuthorDate: Fri Jun 6 03:05:44 2025 -0700
more lenient array expression casting, fix inconsistencies and simplify json_value (#18078)
changes:
* the cast expression will now be more lenient when casting arrays of primitive values. Previously single element arrays were allowed to be cast to scalar values, and more than 1 element would result in a type cast exception. Casting multi-element arrays will now create a null value in the target type instead of failing to be more consistent with other primitive value behavior, which return null if the cast cannot be completed. Complex value casting will still throw type cast exceptions.
* NestedFieldVirtualColumn has been simplified, removing many of its own selector implementations with built in casting in favor of delegating casts to expression selectors when casts are required
* fix several places in NestedFieldVirtualColumn which would allow creating selectors that were not consistent with the expected output type of the virtual column, leading to inconsistent query results depending on what kind of selector was created for the underlying field.
* NestedFieldVirtualColumn now uses JSON_VALUE and JSON_QUERY expressions when no underlying json columns are available to optimize (e.g. when using realtime queries), these selectors were very similar
* Adjusted test data to cover more of the inconsistencies and fixed tests which were encoding incorrect behavior
---
.../druid/msq/exec/MSQComplexGroupByTest.java | 66 +-
.../java/org/apache/druid/math/expr/ExprEval.java | 6 +-
.../java/org/apache/druid/math/expr/Parser.java | 16 +
.../druid/segment/AutoTypeColumnIndexer.java | 5 +-
.../apache/druid/segment/AutoTypeColumnMerger.java | 4 +-
.../apache/druid/segment/column/ColumnType.java | 9 +
.../nested/CompressedNestedDataComplexColumn.java | 9 +-
.../apache/druid/segment/nested/FieldTypeInfo.java | 5 +-
.../nested/NestedFieldDictionaryEncodedColumn.java | 18 +-
.../apache/druid/segment/nested/VariantColumn.java | 5 +-
.../druid/segment/virtual/ExpressionSelectors.java | 170 ++-
.../segment/virtual/ExpressionVectorSelectors.java | 21 +-
.../segment/virtual/NestedFieldVirtualColumn.java | 1319 ++++++++------------
.../java/org/apache/druid/math/expr/EvalTest.java | 73 +-
.../org/apache/druid/math/expr/ExprEvalTest.java | 13 +-
.../druid/query/scan/NestedDataScanQueryTest.java | 2 +-
.../java/org/apache/druid/segment/TestHelper.java | 4 +-
.../segment/virtual/ExpressionSelectorsTest.java | 408 ++++++
.../test/resources/nested-all-types-test-data.json | 2 +-
.../druid/sql/calcite/BaseCalciteQueryTest.java | 2 +-
.../sql/calcite/CalciteNestedDataQueryTest.java | 340 ++++-
21 files changed, 1435 insertions(+), 1062 deletions(-)
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
index db90d4a8a1e..3c0dd86b7f6 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java
@@ -168,22 +168,6 @@ public class MSQComplexGroupByTest extends MSQTestBase
),
1L
},
- new Object[]{
- 0L,
- StructuredData.wrap(
- Map.of(
- "a", 100,
- "b", Map.of(
- "x", "a",
- "y", 1.1,
- "z", List.of(1, 2, 3, 4)
- ),
- "c", 100,
- "v", Collections.emptyList()
- )
- ),
- 1L
- },
new Object[]{
0L,
StructuredData.wrap(
@@ -247,6 +231,22 @@ public class MSQComplexGroupByTest extends MSQTestBase
),
1L
},
+ new Object[]{
+ 0L,
+ StructuredData.wrap(
+ Map.of(
+ "a", 100,
+ "b", Map.of(
+ "x", "a",
+ "y", 1.1,
+ "z", List.of(1, 2, 3, 4)
+ ),
+ "c", List.of(100),
+ "v", Collections.emptyList()
+ )
+ ),
+ 1L
+ },
new Object[]{
0L,
StructuredData.wrap(Map.of("a", 300)),
@@ -303,22 +303,6 @@ public class MSQComplexGroupByTest extends MSQTestBase
),
1L
},
- new Object[]{
- 0L,
- StructuredData.wrap(
- Map.of(
- "a", 100,
- "b", Map.of(
- "x", "a",
- "y", 1.1,
- "z", List.of(1, 2, 3, 4)
- ),
- "c", 100,
- "v", Collections.emptyList()
- )
- ),
- 1L
- },
new Object[]{
0L,
StructuredData.wrap(
@@ -382,6 +366,22 @@ public class MSQComplexGroupByTest extends MSQTestBase
),
1L
},
+ new Object[]{
+ 0L,
+ StructuredData.wrap(
+ Map.of(
+ "a", 100,
+ "b", Map.of(
+ "x", "a",
+ "y", 1.1,
+ "z", List.of(1, 2, 3, 4)
+ ),
+ "c", List.of(100),
+ "v", Collections.emptyList()
+ )
+ ),
+ 1L
+ },
new Object[]{
0L,
StructuredData.wrap(Map.of("a", 300)),
@@ -435,11 +435,11 @@ public class MSQComplexGroupByTest extends MSQTestBase
.setQueryContext(context)
.setExpectedResultRows(List.of(
new Object[]{"{\"a\":600,\"b\":{\"x\":\"f\",\"y\":1.1,\"z\":[6,7,8,9]},\"c\":12.3,\"v\":\"b\"}"},
- new Object[]{"{\"a\":100,\"b\":{\"x\":\"a\",\"y\":1.1,\"z\":[1,2,3,4]},\"c\":100,\"v\":[]}"},
new Object[]{"{\"a\":200,\"b\":{\"x\":\"b\",\"y\":1.1,\"z\":[2,4,6]},\"c\":[\"a\",\"b\"],\"v\":[]}"},
new Object[]{"{\"a\":400,\"b\":{\"x\":\"d\",\"y\":1.1,\"z\":[3,4]},\"c\":{\"a\":1},\"v\":[]}"},
new Object[]{"{\"a\":500,\"b\":{\"x\":\"e\",\"z\":[1,2,3,4]},\"c\":\"hello\",\"v\":\"a\"}"},
new Object[]{"{\"a\":700,\"b\":{\"x\":\"g\",\"y\":1.1,\"z\":[9,null,9,9]},\"c\":null,\"v\":[]}"},
+ new Object[]{"{\"a\":100,\"b\":{\"x\":\"a\",\"y\":1.1,\"z\":[1,2,3,4]},\"c\":[100],\"v\":[]}"},
new Object[]{"{\"a\":300}"}
))
.verifyResults();
diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
index 3634734bfe1..a755d7577eb 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
@@ -1309,17 +1309,17 @@ public abstract class ExprEval<T>
if (value.length == 1) {
return ExprEval.of(asString());
}
- break;
+ return ExprEval.ofType(castTo, null);
case LONG:
if (value.length == 1) {
return isNumericNull() ? ExprEval.ofLong(null) : ExprEval.ofLong(asLong());
}
- break;
+ return ExprEval.ofType(castTo, null);
case DOUBLE:
if (value.length == 1) {
return isNumericNull() ? ExprEval.ofDouble(null) : ExprEval.ofDouble(asDouble());
}
- break;
+ return ExprEval.ofType(castTo, null);
case ARRAY:
ExpressionType elementType = (ExpressionType) castTo.getElementType();
Object[] cast = new Object[value.length];
diff --git a/processing/src/main/java/org/apache/druid/math/expr/Parser.java b/processing/src/main/java/org/apache/druid/math/expr/Parser.java
index e92dd130b7c..fd42daa30fd 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/Parser.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/Parser.java
@@ -144,6 +144,22 @@ public class Parser
return withFlatten ? flatten(parsed) : parsed;
}
+ /**
+ * Create an {@link IdentifierExpr} for some identifier
+ */
+ public static Expr identifier(String identifier)
+ {
+ return new IdentifierExpr(identifier);
+ }
+
+ /**
+ * Create a {@link StringExpr} for a string constant
+ */
+ public static Expr constant(String constant)
+ {
+ return new StringExpr(constant);
+ }
+
/**
* Flatten an {@link Expr}, evaluating expressions on constants where possible to simplify the {@link Expr}.
*/
diff --git a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java
index 28b679d49e2..3f0a6171734 100644
--- a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java
+++ b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java
@@ -444,10 +444,7 @@ public class AutoTypeColumnIndexer implements DimensionIndexer<StructuredData, S
}
if (fieldIndexers.size() == 1 && fieldIndexers.containsKey(NestedPathFinder.JSON_PATH_ROOT)) {
FieldIndexer rootField = fieldIndexers.get(NestedPathFinder.JSON_PATH_ROOT);
- ColumnType logicalType = null;
- for (ColumnType type : FieldTypeInfo.convertToSet(rootField.getTypes().getByteValue())) {
- logicalType = ColumnType.leastRestrictiveType(logicalType, type);
- }
+ ColumnType logicalType = ColumnType.leastRestrictiveType(FieldTypeInfo.convertToSet(rootField.getTypes().getByteValue()));
if (logicalType != null) {
// special handle empty arrays
if (!rootField.getTypes().hasUntypedArray() || logicalType.isArray()) {
diff --git a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java
index 0f8ea4df145..33882d7de9d 100644
--- a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java
+++ b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java
@@ -244,9 +244,7 @@ public class AutoTypeColumnMerger implements DimensionMergerV9
// pick the least restrictive type for the logical type
isVariantType = true;
variantTypeByte = rootTypes.getByteValue();
- for (ColumnType type : FieldTypeInfo.convertToSet(rootTypes.getByteValue())) {
- logicalType = ColumnType.leastRestrictiveType(logicalType, type);
- }
+ logicalType = ColumnType.leastRestrictiveType(FieldTypeInfo.convertToSet(rootTypes.getByteValue()));
// empty arrays can be missed since they don't have a type, so handle them here
if (!logicalType.isArray() && hasArrays) {
logicalType = ColumnTypeFactory.getInstance().ofArray(logicalType);
diff --git a/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java b/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java
index b670d8f4370..c136414158a 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/ColumnType.java
@@ -235,4 +235,13 @@ public class ColumnType extends BaseTypeSignature<ValueType>
return ColumnType.DOUBLE;
}
+ @Nullable
+ public static ColumnType leastRestrictiveType(Iterable<ColumnType> types)
+ {
+ ColumnType leastRestrictiveType = null;
+ for (ColumnType type : types) {
+ leastRestrictiveType = leastRestrictiveType(leastRestrictiveType, type);
+ }
+ return leastRestrictiveType;
+ }
}
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
index a7c6f9df921..96c7258333b 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java
@@ -900,9 +900,7 @@ public abstract class CompressedNestedDataComplexColumn<TStringDictionary extend
}
} else {
fieldTypes = FieldTypeInfo.convertToSet(fieldInfo.getTypes(index).getByteValue());
- for (ColumnType type : fieldTypes) {
- leastRestrictiveType = ColumnType.leastRestrictiveType(leastRestrictiveType, type);
- }
+ leastRestrictiveType = ColumnType.leastRestrictiveType(fieldTypes);
}
return leastRestrictiveType;
}
@@ -1023,7 +1021,10 @@ public abstract class CompressedNestedDataComplexColumn<TStringDictionary extend
ints = VSizeColumnarInts.readFromByteBuffer(dataBuffer);
}
ColumnType theType = types.getSingleType();
- columnBuilder.setType(theType == null ? ColumnType.STRING : theType);
+ if (theType == null) {
+ theType = ColumnType.leastRestrictiveType(FieldTypeInfo.convertToSet(types.getByteValue()));
+ }
+ columnBuilder.setType(theType);
GenericIndexed<ImmutableBitmap> rBitmaps = GenericIndexed.read(
dataBuffer,
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/FieldTypeInfo.java b/processing/src/main/java/org/apache/druid/segment/nested/FieldTypeInfo.java
index b00af7a132f..0ab7f613acf 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/FieldTypeInfo.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/FieldTypeInfo.java
@@ -254,10 +254,7 @@ public class FieldTypeInfo
// adjust for empty array if needed
if (types.hasUntypedArray()) {
Set<ColumnType> columnTypes = FieldTypeInfo.convertToSet(types.getByteValue());
- ColumnType leastRestrictive = null;
- for (ColumnType type : columnTypes) {
- leastRestrictive = ColumnType.leastRestrictiveType(leastRestrictive, type);
- }
+ ColumnType leastRestrictive = ColumnType.leastRestrictiveType(columnTypes);
if (leastRestrictive == null) {
typeByte = add(typeByte, ColumnType.LONG_ARRAY);
} else if (!leastRestrictive.isArray()) {
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldDictionaryEncodedColumn.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldDictionaryEncodedColumn.java
index 09877ab4c69..11b21adbf9d 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldDictionaryEncodedColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldDictionaryEncodedColumn.java
@@ -113,11 +113,7 @@ public class NestedFieldDictionaryEncodedColumn<TStringDictionary extends Indexe
)
{
this.types = types;
- ColumnType leastRestrictive = null;
- for (ColumnType type : FieldTypeInfo.convertToSet(types.getByteValue())) {
- leastRestrictive = ColumnType.leastRestrictiveType(leastRestrictive, type);
- }
- this.logicalType = leastRestrictive;
+ this.logicalType = ColumnType.leastRestrictiveType(FieldTypeInfo.convertToSet(types.getByteValue()));
this.logicalExpressionType = ExpressionType.fromColumnTypeStrict(logicalType);
this.singleType = types.getSingleType();
this.longsColumn = longsColumn;
@@ -163,11 +159,11 @@ public class NestedFieldDictionaryEncodedColumn<TStringDictionary extends Indexe
public String lookupName(int id)
{
final int globalId = dictionary.get(id);
- if (globalId < globalDictionary.size()) {
+ if (globalId < adjustLongId) {
return StringUtils.fromUtf8Nullable(globalDictionary.get(globalId));
- } else if (globalId < globalDictionary.size() + globalLongDictionary.size()) {
+ } else if (globalId < adjustDoubleId) {
return String.valueOf(globalLongDictionary.get(globalId - adjustLongId));
- } else if (globalId < globalDictionary.size() + globalLongDictionary.size() + globalDoubleDictionary.size()) {
+ } else if (globalId < adjustArrayId) {
return String.valueOf(globalDoubleDictionary.get(globalId - adjustDoubleId));
}
return null;
@@ -251,11 +247,11 @@ public class NestedFieldDictionaryEncodedColumn<TStringDictionary extends Indexe
private Object lookupGlobalScalarObject(int globalId)
{
- if (globalId < globalDictionary.size()) {
+ if (globalId < adjustLongId) {
return StringUtils.fromUtf8Nullable(globalDictionary.get(globalId));
- } else if (globalId < globalDictionary.size() + globalLongDictionary.size()) {
+ } else if (globalId < adjustDoubleId) {
return globalLongDictionary.get(globalId - adjustLongId);
- } else if (globalId < globalDictionary.size() + globalLongDictionary.size() + globalDoubleDictionary.size()) {
+ } else if (globalId < adjustArrayId) {
return globalDoubleDictionary.get(globalId - adjustDoubleId);
}
throw new IllegalArgumentException("not a scalar in the dictionary");
diff --git a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumn.java b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumn.java
index 4dd8feca998..6014db28071 100644
--- a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumn.java
@@ -126,10 +126,7 @@ public class VariantColumn<TStringDictionary extends Indexed<ByteBuffer>>
// use the variant type bytes if set, in current code the logical type should have been computed via this same means
// however older versions of the code had a bug which could incorrectly classify mixed types as nested data
if (variantTypeSetByte != null) {
- ColumnType theType = null;
- for (ColumnType type : FieldTypeInfo.convertToSet(variantTypeSetByte)) {
- theType = ColumnType.leastRestrictiveType(theType, type);
- }
+ ColumnType theType = ColumnType.leastRestrictiveType(FieldTypeInfo.convertToSet(variantTypeSetByte));
if (theType != null) {
// sign bit is used to indicate empty arrays, this
if (variantTypeSetByte < 0 && !theType.isArray()) {
diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
index 803dffb2f09..fd7d9db2961 100644
--- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
+++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java
@@ -73,56 +73,7 @@ public class ExpressionSelectors
{
final ColumnValueSelector<ExprEval> baseSelector = makeExprEvalSelector(columnSelectorFactory, expression);
- return new ColumnValueSelector()
- {
- @Override
- public double getDouble()
- {
- // No Assert for null handling as baseSelector already have it.
- return baseSelector.getDouble();
- }
-
- @Override
- public float getFloat()
- {
- // No Assert for null handling as baseSelector already have it.
- return baseSelector.getFloat();
- }
-
- @Override
- public long getLong()
- {
- // No Assert for null handling as baseSelector already have it.
- return baseSelector.getLong();
- }
-
- @Override
- public boolean isNull()
- {
- return baseSelector.isNull();
- }
-
- @Nullable
- @Override
- public Object getObject()
- {
- // No need for null check on getObject() since baseSelector impls will never return null.
- ExprEval eval = baseSelector.getObject();
- return eval.valueOrDefault();
- }
-
- @Override
- public Class classOfObject()
- {
- return Object.class;
- }
-
- @Override
- public void inspectRuntimeShape(RuntimeShapeInspector inspector)
- {
- inspector.visit("baseSelector", baseSelector);
- }
- };
+ return new EvalUnwrappingColumnValueSelector(baseSelector);
}
public static ColumnValueSelector makeStringColumnValueSelector(
@@ -132,35 +83,8 @@ public class ExpressionSelectors
{
final ColumnValueSelector<ExprEval> baseSelector = makeExprEvalSelector(columnSelectorFactory, expression);
- return new ColumnValueSelector()
+ return new EvalUnwrappingColumnValueSelector(baseSelector)
{
- @Override
- public double getDouble()
- {
- // No Assert for null handling as baseSelector already have it.
- return baseSelector.getDouble();
- }
-
- @Override
- public float getFloat()
- {
- // No Assert for null handling as baseSelector already have it.
- return baseSelector.getFloat();
- }
-
- @Override
- public long getLong()
- {
- // No Assert for null handling as baseSelector already have it.
- return baseSelector.getLong();
- }
-
- @Override
- public boolean isNull()
- {
- return baseSelector.isNull();
- }
-
@Nullable
@Override
public Object getObject()
@@ -169,19 +93,34 @@ public class ExpressionSelectors
ExprEval eval = baseSelector.getObject();
return coerceEvalToObjectOrList(eval);
}
+ };
+ }
+ /**
+ * Wrap the output of {@link ColumnValueSelector#getObject()} in {@link ExprEval#ofType(ExpressionType, Object)}
+ * using the supplied delegate type and cast using {@link ExprEval#castTo} to convert to the target type. If the
+ * delegate selector type is null, {@link ExprEval#ofType} falls back to using {@link ExprEval#bestEffortOf}, however
+ * the target type must be not null, or else this method will fail when attempting to convert to an expression type.
+ */
+ public static ColumnValueSelector<?> castColumnValueSelector(
+ RowIdSupplier rowIdSupplier,
+ ColumnValueSelector<?> delegate,
+ @Nullable ColumnType delegateType,
+ ColumnType castToType
+ )
+ {
+ final ExpressionType fromType = ExpressionType.fromColumnType(delegateType);
+ final ExpressionType toType = ExpressionType.fromColumnTypeStrict(castToType);
+ final ColumnValueSelector<ExprEval> cast = new BaseExpressionColumnValueSelector(rowIdSupplier)
+ {
@Override
- public Class classOfObject()
- {
- return Object.class;
- }
-
- @Override
- public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ protected ExprEval<?> eval()
{
- inspector.visit("baseSelector", baseSelector);
+ return ExprEval.ofType(fromType, delegate.getObject()).castTo(toType);
}
};
+
+ return new EvalUnwrappingColumnValueSelector(cast);
}
/**
@@ -534,4 +473,63 @@ public class ExpressionSelectors
}
return eval.valueOrDefault();
}
+
+ /**
+ * Wraps a {@link ColumnValueSelector<ExprEval>} and calls {@link ExprEval#valueOrDefault()} on the output of
+ * {@link #baseSelector#getObject()} in {@link #getObject()}.
+ */
+ private static class EvalUnwrappingColumnValueSelector implements ColumnValueSelector
+ {
+ private final ColumnValueSelector<ExprEval> baseSelector;
+
+ public EvalUnwrappingColumnValueSelector(ColumnValueSelector<ExprEval> baseSelector)
+ {
+ this.baseSelector = baseSelector;
+ }
+
+ @Override
+ public double getDouble()
+ {
+ return baseSelector.getDouble();
+ }
+
+ @Override
+ public float getFloat()
+ {
+ return baseSelector.getFloat();
+ }
+
+ @Override
+ public long getLong()
+ {
+ return baseSelector.getLong();
+ }
+
+ @Override
+ public boolean isNull()
+ {
+ return baseSelector.isNull();
+ }
+
+ @Nullable
+ @Override
+ public Object getObject()
+ {
+ // No need for null check on getObject() since baseSelector impls will never return null.
+ ExprEval eval = baseSelector.getObject();
+ return eval.valueOrDefault();
+ }
+
+ @Override
+ public Class classOfObject()
+ {
+ return Object.class;
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+ inspector.visit("baseSelector", baseSelector);
+ }
+ }
}
diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java
index 9c1d6c2c941..cb103f424f4 100644
--- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java
+++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java
@@ -179,7 +179,7 @@ public class ExpressionVectorSelectors
ColumnType castTo
)
{
- ExpressionVectorInputBinding binding = new ExpressionVectorInputBinding(inspector);
+ final ExpressionVectorInputBinding binding = new ExpressionVectorInputBinding(inspector);
binding.addNumeric(columnName, ExpressionType.fromColumnType(selectorType), selector);
return new ExpressionVectorObjectSelector(
CastToTypeVectorProcessor.cast(
@@ -190,6 +190,25 @@ public class ExpressionVectorSelectors
);
}
+ public static VectorObjectSelector castObject(
+ ReadableVectorInspector inspector,
+ String columnName,
+ VectorObjectSelector selector,
+ ColumnType selectorType,
+ ColumnType castTo
+ )
+ {
+ final ExpressionVectorInputBinding binding = new ExpressionVectorInputBinding(inspector);
+ binding.addObjectSelector(columnName, ExpressionType.fromColumnType(selectorType), selector);
+ return new ExpressionVectorObjectSelector(
+ CastToTypeVectorProcessor.cast(
+ VectorProcessors.identifier(binding, columnName),
+ ExpressionType.fromColumnType(castTo)
+ ),
+ binding
+ );
+ }
+
public static VectorValueSelector castObjectSelectorToNumeric(
ReadableVectorInspector inspector,
String columnName,
diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
index f222990fe2e..3429975b94c 100644
--- a/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
+++ b/processing/src/main/java/org/apache/druid/segment/virtual/NestedFieldVirtualColumn.java
@@ -25,14 +25,15 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Doubles;
-import org.apache.druid.common.guava.GuavaUtils;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.Numbers;
-import org.apache.druid.math.expr.Evals;
+import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
import org.apache.druid.math.expr.ExpressionType;
+import org.apache.druid.math.expr.Parser;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.dimension.DimensionSpec;
+import org.apache.druid.query.expression.NestedDataExpressions;
import org.apache.druid.query.extraction.ExtractionFn;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.query.filter.DruidPredicateFactory;
@@ -108,6 +109,9 @@ import java.util.Set;
*/
public class NestedFieldVirtualColumn implements VirtualColumn
{
+ private static final NestedDataExpressions.JsonQueryExprMacro JSON_QUERY = new NestedDataExpressions.JsonQueryExprMacro();
+ private static final NestedDataExpressions.JsonValueExprMacro JSON_VALUE = new NestedDataExpressions.JsonValueExprMacro();
+
private final String outputName;
private final NestedFieldSpec fieldSpec;
@@ -177,6 +181,7 @@ public class NestedFieldVirtualColumn implements VirtualColumn
this(columnName, outputName, expectedType, null, null, path, false);
}
+ @Nullable
@Override
public byte[] getCacheKey()
{
@@ -208,6 +213,7 @@ public class NestedFieldVirtualColumn implements VirtualColumn
return fieldSpec.parts;
}
+ @Nullable
@JsonProperty
public ColumnType getExpectedType()
{
@@ -239,14 +245,27 @@ public class NestedFieldVirtualColumn implements VirtualColumn
ColumnSelectorFactory factory
)
{
- // this column value selector is used for realtime queries, so we always process StructuredData
- final ColumnValueSelector<?> baseSelector = factory.makeColumnValueSelector(fieldSpec.columnName);
-
- // processFromRaw is true that means JSON_QUERY, which can return partial results, otherwise this virtual column
- // is JSON_VALUE which only returns literals, so use the literal value selector instead
- return fieldSpec.processFromRaw
- ? new RawFieldColumnSelector(baseSelector, fieldSpec.parts)
- : new RawFieldLiteralColumnValueSelector(baseSelector, fieldSpec.parts);
+ // realtime selectors have no optimization, fallback to json_query/json_value expressions
+ final Expr identifier = Parser.identifier(fieldSpec.columnName);
+ final Expr path = Parser.constant(NestedPathFinder.toNormalizedJsonPath(fieldSpec.parts));
+ final Expr jsonExpr;
+ if (fieldSpec.processFromRaw) {
+ // processFromRaw is true that means JSON_QUERY, which can return partial object results
+ jsonExpr = JSON_QUERY.apply(List.of(identifier, path));
+ } else {
+ // otherwise, this virtual column is JSON_VALUE which only returns primitives
+ final List<Expr> args;
+ if (fieldSpec.expectedType != null) {
+ final Expr castType = Parser.constant(
+ ExpressionType.fromColumnTypeStrict(fieldSpec.expectedType).asTypeString()
+ );
+ args = List.of(identifier, path, castType);
+ } else {
+ args = List.of(identifier, path);
+ }
+ jsonExpr = JSON_VALUE.apply(args);
+ }
+ return ExpressionSelectors.makeColumnValueSelector(factory, jsonExpr);
}
@Nullable
@@ -282,6 +301,10 @@ public class NestedFieldVirtualColumn implements VirtualColumn
BaseColumn theColumn = holder.getColumn();
if (theColumn instanceof NestedDataComplexColumn) {
final NestedDataComplexColumn column = (NestedDataComplexColumn) theColumn;
+ final ColumnType logicalType = column.getFieldLogicalType(fieldSpec.parts);
+ if (logicalType != null && logicalType.isArray()) {
+ return new FieldDimensionSelector(column.makeColumnValueSelector(fieldSpec.parts, offset));
+ }
return column.makeDimensionSelector(fieldSpec.parts, offset, extractionFn);
}
@@ -302,7 +325,7 @@ public class NestedFieldVirtualColumn implements VirtualColumn
);
}
- if (fieldSpec.parts.size() == 1 && fieldSpec.parts.get(0) instanceof NestedPathArrayElement && theColumn instanceof VariantColumn) {
+ if (isRootArrayElementPathAndArrayColumn(theColumn)) {
final VariantColumn<?> arrayColumn = (VariantColumn<?>) theColumn;
ColumnValueSelector<?> arraySelector = arrayColumn.makeColumnValueSelector(offset);
final int elementNumber = ((NestedPathArrayElement) fieldSpec.parts.get(0)).getIndex();
@@ -364,8 +387,18 @@ public class NestedFieldVirtualColumn implements VirtualColumn
}
// "JSON_VALUE", which only returns literals, on a NestedDataComplexColumn, so we can use the fields value selector
+
if (theColumn instanceof NestedDataComplexColumn) {
final NestedDataComplexColumn column = (NestedDataComplexColumn) theColumn;
+ final ColumnType fieldType = column.getFieldLogicalType(fieldSpec.parts);
+ if (fieldType != null && fieldSpec.expectedType != null && !fieldSpec.expectedType.equals(fieldType)) {
+ return ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ column.makeColumnValueSelector(fieldSpec.parts, offset),
+ fieldType,
+ fieldSpec.expectedType
+ );
+ }
return column.makeColumnValueSelector(fieldSpec.parts, offset);
}
@@ -382,69 +415,14 @@ public class NestedFieldVirtualColumn implements VirtualColumn
return theColumn.makeColumnValueSelector(offset);
}
- if (fieldSpec.parts.size() == 1 && fieldSpec.parts.get(0) instanceof NestedPathArrayElement && theColumn instanceof VariantColumn) {
+ if (isRootArrayElementPathAndArrayColumn(theColumn)) {
final VariantColumn<?> arrayColumn = (VariantColumn<?>) theColumn;
ColumnValueSelector<?> arraySelector = arrayColumn.makeColumnValueSelector(offset);
final int elementNumber = ((NestedPathArrayElement) fieldSpec.parts.get(0)).getIndex();
if (elementNumber < 0) {
throw new IAE("Cannot make array element selector, negative array index not supported");
}
- return new ColumnValueSelector<>()
- {
- @Override
- public boolean isNull()
- {
- Object o = getObject();
- return !(o instanceof Number);
- }
-
- @Override
- public long getLong()
- {
- Object o = getObject();
- return o instanceof Number ? ((Number) o).longValue() : 0L;
- }
-
- @Override
- public float getFloat()
- {
- Object o = getObject();
- return o instanceof Number ? ((Number) o).floatValue() : 0f;
- }
-
- @Override
- public double getDouble()
- {
- Object o = getObject();
- return o instanceof Number ? ((Number) o).doubleValue() : 0.0;
- }
-
- @Override
- public void inspectRuntimeShape(RuntimeShapeInspector inspector)
- {
- arraySelector.inspectRuntimeShape(inspector);
- }
-
- @Nullable
- @Override
- public Object getObject()
- {
- Object o = arraySelector.getObject();
- if (o instanceof Object[]) {
- Object[] array = (Object[]) o;
- if (elementNumber < array.length) {
- return array[elementNumber];
- }
- }
- return null;
- }
-
- @Override
- public Class<?> classOfObject()
- {
- return Object.class;
- }
- };
+ return new ArrayElementColumnValueSelector(arraySelector, elementNumber);
}
// we are not a nested column and are being asked for a path that will never exist, so we are nil selector
@@ -529,17 +507,8 @@ public class NestedFieldVirtualColumn implements VirtualColumn
);
}
final VectorObjectSelector objectSelector = complexColumn.makeVectorObjectSelector(fieldSpec.parts, offset);
- if (leastRestrictiveType != null &&
- leastRestrictiveType.isArray() &&
- fieldSpec.expectedType != null &&
- !fieldSpec.expectedType.isArray()
- ) {
- final ExpressionType elementType = ExpressionType.fromColumnTypeStrict(leastRestrictiveType.getElementType());
- final ExpressionType castTo = ExpressionType.fromColumnTypeStrict(fieldSpec.expectedType);
- return makeVectorArrayToScalarObjectSelector(offset, objectSelector, elementType, castTo);
- }
- return objectSelector;
+ return castVectorObjectSelectorIfNeeded(columnName, offset, leastRestrictiveType, objectSelector);
}
// not a nested column, but we can still do stuff if the path is the 'root', indicated by an empty path parts
if (fieldSpec.parts.isEmpty()) {
@@ -559,19 +528,11 @@ public class NestedFieldVirtualColumn implements VirtualColumn
fieldSpec.expectedType
);
}
- // if the underlying column is array typed, the vector object selector it spits out will homogenize stuff to
- // make all of the objects a consistent type, which is typically a good thing, but if we are doing mixed type
- // stuff and expect the output type to be scalar typed, then we should coerce things to only extract the scalars
- if (capabilities.isArray() && !fieldSpec.expectedType.isArray()) {
- final VectorObjectSelector delegate = column.makeVectorObjectSelector(offset);
- final ExpressionType elementType = ExpressionType.fromColumnTypeStrict(capabilities.getElementType());
- final ExpressionType castTo = ExpressionType.fromColumnTypeStrict(fieldSpec.expectedType);
- return makeVectorArrayToScalarObjectSelector(offset, delegate, elementType, castTo);
- }
- return column.makeVectorObjectSelector(offset);
+ final VectorObjectSelector delegate = column.makeVectorObjectSelector(offset);
+ return castVectorObjectSelectorIfNeeded(columnName, offset, capabilities.toColumnType(), delegate);
}
- if (fieldSpec.parts.size() == 1 && fieldSpec.parts.get(0) instanceof NestedPathArrayElement && column instanceof VariantColumn) {
+ if (isRootArrayElementPathAndArrayColumn(column)) {
final VariantColumn<?> arrayColumn = (VariantColumn<?>) column;
final ExpressionType elementType = ExpressionType.fromColumnTypeStrict(
arrayColumn.getLogicalType().isArray() ? arrayColumn.getLogicalType().getElementType() : arrayColumn.getLogicalType()
@@ -584,46 +545,7 @@ public class NestedFieldVirtualColumn implements VirtualColumn
if (elementNumber < 0) {
throw new IAE("Cannot make array element selector, negative array index not supported");
}
- return new VectorObjectSelector()
- {
- private final Object[] elements = new Object[arraySelector.getMaxVectorSize()];
- private int id = ReadableVectorInspector.NULL_ID;
-
- @Override
- public Object[] getObjectVector()
- {
- if (offset.getId() != id) {
- final Object[] delegate = arraySelector.getObjectVector();
- for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
- Object maybeArray = delegate[i];
- if (maybeArray instanceof Object[]) {
- Object[] anArray = (Object[]) maybeArray;
- if (elementNumber < anArray.length) {
- elements[i] = ExprEval.ofType(elementType, anArray[elementNumber]).castTo(castTo).value();
- } else {
- elements[i] = null;
- }
- } else {
- elements[i] = null;
- }
- }
- id = offset.getId();
- }
- return elements;
- }
-
- @Override
- public int getMaxVectorSize()
- {
- return arraySelector.getMaxVectorSize();
- }
-
- @Override
- public int getCurrentVectorSize()
- {
- return arraySelector.getCurrentVectorSize();
- }
- };
+ return new ArrayElementVectorObjectSelector(arraySelector, offset, elementNumber, elementType, castTo);
}
// we are not a nested column and are being asked for a path that will never exist, so we are nil selector
@@ -652,147 +574,20 @@ public class NestedFieldVirtualColumn implements VirtualColumn
// ... revisit this if that ever changes)
if (theColumn instanceof DictionaryEncodedColumn) {
final VectorObjectSelector delegate = theColumn.makeVectorObjectSelector(offset);
- if (fieldSpec.expectedType != null && fieldSpec.expectedType.is(ValueType.LONG)) {
- return new BaseLongVectorValueSelector(offset)
- {
- private int currentOffsetId = ReadableVectorInspector.NULL_ID;
- private final long[] longs = new long[delegate.getMaxVectorSize()];
- @Nullable
- private boolean[] nulls = null;
-
- @Override
- public long[] getLongVector()
- {
- computeLongs();
- return longs;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- computeLongs();
- return nulls;
- }
-
- private void computeLongs()
- {
- if (currentOffsetId != offset.getId()) {
- currentOffsetId = offset.getId();
- final Object[] values = delegate.getObjectVector();
- for (int i = 0; i < values.length; i++) {
- Number n = ExprEval.computeNumber(Evals.asString(values[i]));
- if (n != null) {
- longs[i] = n.longValue();
- if (nulls != null) {
- nulls[i] = false;
- }
- } else {
- if (nulls == null) {
- nulls = new boolean[offset.getMaxVectorSize()];
- }
- nulls[i] = true;
- }
- }
- }
- }
- };
- } else if (fieldSpec.expectedType != null && fieldSpec.expectedType.is(ValueType.FLOAT)) {
- return new BaseFloatVectorValueSelector(offset)
- {
- private int currentOffsetId = ReadableVectorInspector.NULL_ID;
- private final float[] floats = new float[delegate.getMaxVectorSize()];
- @Nullable
- private boolean[] nulls = null;
-
- @Override
- public float[] getFloatVector()
- {
- computeFloats();
- return floats;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- computeFloats();
- return nulls;
- }
-
- private void computeFloats()
- {
- if (currentOffsetId != offset.getId()) {
- currentOffsetId = offset.getId();
- final Object[] values = delegate.getObjectVector();
- for (int i = 0; i < values.length; i++) {
- Number n = ExprEval.computeNumber(Evals.asString(values[i]));
- if (n != null) {
- floats[i] = n.floatValue();
- if (nulls != null) {
- nulls[i] = false;
- }
- } else {
- if (nulls == null) {
- nulls = new boolean[offset.getMaxVectorSize()];
- }
- nulls[i] = true;
- }
- }
- }
- }
- };
- } else {
- return new BaseDoubleVectorValueSelector(offset)
- {
- private int currentOffsetId = ReadableVectorInspector.NULL_ID;
- private final double[] doubles = new double[delegate.getMaxVectorSize()];
- @Nullable
- private boolean[] nulls = null;
- @Override
- public double[] getDoubleVector()
- {
- computeDoubles();
- return doubles;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- computeDoubles();
- return nulls;
- }
-
- private void computeDoubles()
- {
- if (currentOffsetId != offset.getId()) {
- currentOffsetId = offset.getId();
- final Object[] values = delegate.getObjectVector();
- for (int i = 0; i < values.length; i++) {
- Number n = ExprEval.computeNumber(Evals.asString(values[i]));
- if (n != null) {
- doubles[i] = n.doubleValue();
- if (nulls != null) {
- nulls[i] = false;
- }
- } else {
- if (nulls == null) {
- nulls = new boolean[offset.getMaxVectorSize()];
- }
- nulls[i] = true;
- }
- }
- }
- }
- };
- }
+ final ColumnType castTo = fieldSpec.expectedType != null ? fieldSpec.expectedType : ColumnType.DOUBLE;
+ return ExpressionVectorSelectors.castObjectSelectorToNumeric(
+ offset,
+ columnName,
+ delegate,
+ holder.getCapabilities().toColumnType(),
+ castTo
+ );
}
// otherwise, just use the columns native vector value selector (this might explode if not natively numeric)
return theColumn.makeVectorValueSelector(offset);
}
// array columns can also be handled if the path is a root level array element accessor
- if (fieldSpec.parts.size() == 1 && fieldSpec.parts.get(0) instanceof NestedPathArrayElement && theColumn instanceof VariantColumn) {
+ if (isRootArrayElementPathAndArrayColumn(theColumn)) {
final VariantColumn<?> arrayColumn = (VariantColumn<?>) theColumn;
VectorObjectSelector arraySelector = arrayColumn.makeVectorObjectSelector(offset);
final int elementNumber = ((NestedPathArrayElement) fieldSpec.parts.get(0)).getIndex();
@@ -801,210 +596,11 @@ public class NestedFieldVirtualColumn implements VirtualColumn
}
if (fieldSpec.expectedType != null && fieldSpec.expectedType.is(ValueType.LONG)) {
- return new BaseLongVectorValueSelector(offset)
- {
- private final long[] longs = new long[offset.getMaxVectorSize()];
- private final boolean[] nulls = new boolean[offset.getMaxVectorSize()];
- private int id = ReadableVectorInspector.NULL_ID;
-
- private void computeNumbers()
- {
- if (offset.getId() != id) {
- final Object[] maybeArrays = arraySelector.getObjectVector();
- for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
- Object maybeArray = maybeArrays[i];
- if (maybeArray instanceof Object[]) {
- Object[] anArray = (Object[]) maybeArray;
- if (elementNumber < anArray.length) {
- if (anArray[elementNumber] instanceof Number) {
- Number n = (Number) anArray[elementNumber];
- longs[i] = n.longValue();
- nulls[i] = false;
- } else {
- Double d = anArray[elementNumber] instanceof String
- ? Doubles.tryParse((String) anArray[elementNumber])
- : null;
- if (d != null) {
- longs[i] = d.longValue();
- nulls[i] = false;
- } else {
- longs[i] = 0L;
- nulls[i] = true;
- }
- }
- } else {
- nullElement(i);
- }
- } else {
- // not an array?
- nullElement(i);
- }
- }
- id = offset.getId();
- }
- }
-
- private void nullElement(int i)
- {
- longs[i] = 0L;
- nulls[i] = true;
- }
-
- @Override
- public long[] getLongVector()
- {
- if (offset.getId() != id) {
- computeNumbers();
- }
- return longs;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- if (offset.getId() != id) {
- computeNumbers();
- }
- return nulls;
- }
- };
+ return new ArrayElementLongVectorValueSelector(offset, arraySelector, elementNumber);
} else if (fieldSpec.expectedType != null && fieldSpec.expectedType.is(ValueType.FLOAT)) {
- return new BaseFloatVectorValueSelector(offset)
- {
- private final float[] floats = new float[offset.getMaxVectorSize()];
- private final boolean[] nulls = new boolean[offset.getMaxVectorSize()];
- private int id = ReadableVectorInspector.NULL_ID;
-
- private void computeNumbers()
- {
- if (offset.getId() != id) {
- final Object[] maybeArrays = arraySelector.getObjectVector();
- for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
- Object maybeArray = maybeArrays[i];
- if (maybeArray instanceof Object[]) {
- Object[] anArray = (Object[]) maybeArray;
- if (elementNumber < anArray.length) {
- if (anArray[elementNumber] instanceof Number) {
- Number n = (Number) anArray[elementNumber];
- floats[i] = n.floatValue();
- nulls[i] = false;
- } else {
- Double d = anArray[elementNumber] instanceof String
- ? Doubles.tryParse((String) anArray[elementNumber])
- : null;
- if (d != null) {
- floats[i] = d.floatValue();
- nulls[i] = false;
- } else {
- nullElement(i);
- }
- }
- } else {
- nullElement(i);
- }
- } else {
- // not an array?
- nullElement(i);
- }
- }
- id = offset.getId();
- }
- }
-
- private void nullElement(int i)
- {
- floats[i] = 0f;
- nulls[i] = true;
- }
-
- @Override
- public float[] getFloatVector()
- {
- if (offset.getId() != id) {
- computeNumbers();
- }
- return floats;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- if (offset.getId() != id) {
- computeNumbers();
- }
- return nulls;
- }
- };
+ return new ArrayElementFloatVectorValueSelector(offset, arraySelector, elementNumber);
} else {
- return new BaseDoubleVectorValueSelector(offset)
- {
- private final double[] doubles = new double[offset.getMaxVectorSize()];
- private final boolean[] nulls = new boolean[offset.getMaxVectorSize()];
- private int id = ReadableVectorInspector.NULL_ID;
-
- private void computeNumbers()
- {
- if (offset.getId() != id) {
- final Object[] maybeArrays = arraySelector.getObjectVector();
- for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
- Object maybeArray = maybeArrays[i];
- if (maybeArray instanceof Object[]) {
- Object[] anArray = (Object[]) maybeArray;
- if (elementNumber < anArray.length) {
- if (anArray[elementNumber] instanceof Number) {
- Number n = (Number) anArray[elementNumber];
- doubles[i] = n.doubleValue();
- nulls[i] = false;
- } else {
- Double d = anArray[elementNumber] instanceof String
- ? Doubles.tryParse((String) anArray[elementNumber])
- : null;
- if (d != null) {
- doubles[i] = d;
- nulls[i] = false;
- } else {
- nullElement(i);
- }
- }
- } else {
- nullElement(i);
- }
- } else {
- // not an array?
- nullElement(i);
- }
- }
- id = offset.getId();
- }
- }
-
- private void nullElement(int i)
- {
- doubles[i] = 0.0;
- nulls[i] = true;
- }
-
- @Override
- public double[] getDoubleVector()
- {
- if (offset.getId() != id) {
- computeNumbers();
- }
- return doubles;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- if (offset.getId() != id) {
- computeNumbers();
- }
- return nulls;
- }
- };
+ return new ArrayElementDoubleVectorValueSelector(offset, arraySelector, elementNumber);
}
}
return NilVectorSelector.create(offset);
@@ -1013,157 +609,20 @@ public class NestedFieldVirtualColumn implements VirtualColumn
final NestedDataComplexColumn column = (NestedDataComplexColumn) theColumn;
// if column is numeric, it has a vector value selector, so we can directly make a vector value selector
// if we are missing an expectedType, then we've got nothing else to work with so try it anyway
+ final ColumnType leastRestrictiveType = column.getFieldLogicalType(fieldSpec.parts);
+
if (column.isNumeric(fieldSpec.parts) || fieldSpec.expectedType == null) {
return column.makeVectorValueSelector(fieldSpec.parts, offset);
}
- final ColumnType leastRestrictiveType = column.getFieldLogicalType(fieldSpec.parts);
final VectorObjectSelector fieldSelector = column.makeVectorObjectSelector(fieldSpec.parts, offset);
- final VectorObjectSelector objectSelector;
- // if the field has array types, wrap the object selector in the array to scalar value coercer
- if (leastRestrictiveType != null && leastRestrictiveType.isArray()) {
- objectSelector = makeVectorArrayToScalarObjectSelector(
- offset,
- fieldSelector,
- ExpressionType.fromColumnTypeStrict(leastRestrictiveType.getElementType()),
- ExpressionType.fromColumnTypeStrict(fieldSpec.expectedType)
- );
- } else {
- objectSelector = fieldSelector;
- }
- if (fieldSpec.expectedType.is(ValueType.LONG)) {
- return new BaseLongVectorValueSelector(offset)
- {
- private final long[] longVector = new long[offset.getMaxVectorSize()];
-
- @Nullable
- private boolean[] nullVector = null;
- private int id = ReadableVectorInspector.NULL_ID;
-
- @Override
- public long[] getLongVector()
- {
- computeVectorsIfNeeded();
- return longVector;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- computeVectorsIfNeeded();
- return nullVector;
- }
-
- private void computeVectorsIfNeeded()
- {
- if (id == offset.getId()) {
- return;
- }
- id = offset.getId();
- final Object[] vals = objectSelector.getObjectVector();
- for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++) {
- Object v = vals[i];
- if (v == null) {
- if (nullVector == null) {
- nullVector = new boolean[objectSelector.getMaxVectorSize()];
- }
- longVector[i] = 0L;
- nullVector[i] = true;
- } else {
- Long l;
- if (v instanceof Number) {
- l = ((Number) v).longValue();
- } else {
- final String s = String.valueOf(v);
- l = GuavaUtils.tryParseLong(s);
- if (l == null) {
- final Double d = Doubles.tryParse(s);
- if (d != null) {
- l = d.longValue();
- }
- }
- }
- if (l != null) {
- longVector[i] = l;
- if (nullVector != null) {
- nullVector[i] = false;
- }
- } else {
- if (nullVector == null) {
- nullVector = new boolean[objectSelector.getMaxVectorSize()];
- }
- longVector[i] = 0L;
- nullVector[i] = true;
- }
- }
- }
- }
- };
- } else {
- // treat anything else as double
- return new BaseDoubleVectorValueSelector(offset)
- {
- private final double[] doubleVector = new double[offset.getMaxVectorSize()];
-
- @Nullable
- private boolean[] nullVector = null;
- private int id = ReadableVectorInspector.NULL_ID;
-
- @Override
- public double[] getDoubleVector()
- {
- computeVectorsIfNeeded();
- return doubleVector;
- }
-
- @Nullable
- @Override
- public boolean[] getNullVector()
- {
- computeVectorsIfNeeded();
- return nullVector;
- }
-
- private void computeVectorsIfNeeded()
- {
- if (id == offset.getId()) {
- return;
- }
- id = offset.getId();
- final Object[] vals = objectSelector.getObjectVector();
- for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++) {
- Object v = vals[i];
- if (v == null) {
- if (nullVector == null) {
- nullVector = new boolean[objectSelector.getMaxVectorSize()];
- }
- doubleVector[i] = 0.0;
- nullVector[i] = true;
- } else {
- Double d;
- if (v instanceof Number) {
- d = ((Number) v).doubleValue();
- } else {
- d = Doubles.tryParse(String.valueOf(v));
- }
- if (d != null) {
- doubleVector[i] = d;
- if (nullVector != null) {
- nullVector[i] = false;
- }
- } else {
- if (nullVector == null) {
- nullVector = new boolean[objectSelector.getMaxVectorSize()];
- }
- doubleVector[i] = 0.0;
- nullVector[i] = true;
- }
- }
- }
- }
- };
- }
+ return ExpressionVectorSelectors.castObjectSelectorToNumeric(
+ offset,
+ columnName,
+ fieldSelector,
+ leastRestrictiveType,
+ fieldSpec.expectedType
+ );
}
@Nullable
@@ -1213,7 +672,7 @@ public class NestedFieldVirtualColumn implements VirtualColumn
}
return baseIndexSupplier;
}
- if (fieldSpec.parts.size() == 1 && fieldSpec.parts.get(0) instanceof NestedPathArrayElement && theColumn instanceof VariantColumn) {
+ if (isRootArrayElementPathAndArrayColumn(theColumn)) {
// cannot use the array column index supplier directly, in the future array columns should expose a function
// with a signature like 'getArrayElementIndexSupplier(int index)' to allow getting indexes for specific elements
// if we want to support this stuff. Right now VariantArrayColumn doesn't actually retain enough information about
@@ -1347,58 +806,32 @@ public class NestedFieldVirtualColumn implements VirtualColumn
}
/**
- * Create a {@link VectorObjectSelector} from a base selector which may return ARRAY types, coercing to some scalar
- * value. Single element arrays will be unwrapped, while multi-element arrays will become null values. Non-arrays
- * will be best effort cast to the castTo type.
+ * Returns true if json path is a root array element (for example '$[1]') and the column array column
*/
- private static VectorObjectSelector makeVectorArrayToScalarObjectSelector(
+ private boolean isRootArrayElementPathAndArrayColumn(BaseColumn theColumn)
+ {
+ return fieldSpec.parts.size() == 1
+ && fieldSpec.parts.get(0) instanceof NestedPathArrayElement
+ && theColumn instanceof VariantColumn;
+ }
+
+ private VectorObjectSelector castVectorObjectSelectorIfNeeded(
+ String columnName,
ReadableVectorOffset offset,
- VectorObjectSelector delegate,
- ExpressionType elementType,
- ExpressionType castTo
+ ColumnType leastRestrictiveType,
+ VectorObjectSelector objectSelector
)
{
- return new VectorObjectSelector()
- {
- final Object[] scalars = new Object[offset.getMaxVectorSize()];
- private int id = ReadableVectorInspector.NULL_ID;
-
- @Override
- public Object[] getObjectVector()
- {
- if (offset.getId() != id) {
- Object[] result = delegate.getObjectVector();
- for (int i = 0; i < offset.getCurrentVectorSize(); i++) {
- if (result[i] instanceof Object[]) {
- Object[] o = (Object[]) result[i];
- if (o == null || o.length != 1) {
- scalars[i] = null;
- } else {
- ExprEval<?> element = ExprEval.ofType(elementType, o[0]);
- scalars[i] = element.castTo(castTo).value();
- }
- } else {
- ExprEval<?> element = ExprEval.bestEffortOf(result[i]);
- scalars[i] = element.castTo(castTo).value();
- }
- }
- id = offset.getId();
- }
- return scalars;
- }
-
- @Override
- public int getMaxVectorSize()
- {
- return offset.getMaxVectorSize();
- }
-
- @Override
- public int getCurrentVectorSize()
- {
- return offset.getCurrentVectorSize();
- }
- };
+ if (fieldSpec.expectedType != null && !Objects.equals(fieldSpec.expectedType, leastRestrictiveType)) {
+ return ExpressionVectorSelectors.castObject(
+ offset,
+ columnName,
+ objectSelector,
+ leastRestrictiveType,
+ fieldSpec.expectedType
+ );
+ }
+ return objectSelector;
}
private static class NestedFieldSpec implements EquivalenceKey
@@ -1445,159 +878,12 @@ public class NestedFieldVirtualColumn implements VirtualColumn
}
}
- /**
- * Process the "raw" data to extract non-complex values. Like {@link RawFieldColumnSelector} but does not return
- * complex nested objects and does not wrap the results in {@link StructuredData}.
- * <p>
- * This is used as a selector on realtime data when the native field columns are not available.
- */
- public static class RawFieldLiteralColumnValueSelector extends RawFieldColumnSelector
- {
- public RawFieldLiteralColumnValueSelector(
- ColumnValueSelector baseSelector,
- List<NestedPathPart> parts
- )
- {
- super(baseSelector, parts);
- }
-
- @Override
- public double getDouble()
- {
- Object o = getObject();
- return Numbers.tryParseDouble(o, 0.0);
- }
-
- @Override
- public float getFloat()
- {
- Object o = getObject();
- return Numbers.tryParseFloat(o, 0.0f);
- }
-
- @Override
- public long getLong()
- {
- Object o = getObject();
- return Numbers.tryParseLong(o, 0L);
- }
-
- @Override
- public boolean isNull()
- {
- final Object o = getObject();
- if (o instanceof Number) {
- return false;
- }
- if (o instanceof String) {
- return GuavaUtils.tryParseLong((String) o) == null && Doubles.tryParse((String) o) == null;
- }
- return true;
- }
-
- @Nullable
- @Override
- public Object getObject()
- {
- final StructuredData data = StructuredData.wrap(baseSelector.getObject());
- if (data == null) {
- return null;
- }
-
- final Object valAtPath = NestedPathFinder.find(data.getValue(), parts);
- final ExprEval eval = ExprEval.bestEffortOf(valAtPath);
- if (eval.type().isPrimitive() || eval.type().isPrimitiveArray()) {
- return eval.valueOrDefault();
- }
- // not a primitive value, return null;
- return null;
- }
-
- }
-
- /**
- * Process the "raw" data to extract values with {@link NestedPathFinder#find(Object, List)}, wrapping the result in
- * {@link StructuredData}
- */
- public static class RawFieldColumnSelector implements ColumnValueSelector<Object>
- {
- protected final ColumnValueSelector baseSelector;
- protected final List<NestedPathPart> parts;
-
- public RawFieldColumnSelector(ColumnValueSelector baseSelector, List<NestedPathPart> parts)
- {
- this.baseSelector = baseSelector;
- this.parts = parts;
- }
-
- @Override
- public double getDouble()
- {
- StructuredData data = (StructuredData) getObject();
- if (data != null) {
- return Numbers.tryParseDouble(data.getValue(), 0.0);
- }
- return 0.0;
- }
-
- @Override
- public float getFloat()
- {
- StructuredData data = (StructuredData) getObject();
- if (data != null) {
- return Numbers.tryParseFloat(data.getValue(), 0f);
- }
- return 0f;
- }
-
- @Override
- public long getLong()
- {
- StructuredData data = (StructuredData) getObject();
- if (data != null) {
- return Numbers.tryParseLong(data.getValue(), 0L);
- }
- return 0L;
- }
-
- @Override
- public void inspectRuntimeShape(RuntimeShapeInspector inspector)
- {
- inspector.visit("baseSelector", baseSelector);
- inspector.visit("parts", parts);
- }
-
- @Override
- public boolean isNull()
- {
- StructuredData data = (StructuredData) getObject();
- if (data == null) {
- return true;
- }
- Object o = data.getValue();
- return !(o instanceof Number || (o instanceof String && Doubles.tryParse((String) o) != null));
- }
-
- @Nullable
- @Override
- public Object getObject()
- {
- StructuredData data = StructuredData.wrap(baseSelector.getObject());
- return StructuredData.wrap(NestedPathFinder.find(data == null ? null : data.getValue(), parts));
- }
-
- @Override
- public Class<?> classOfObject()
- {
- return Object.class;
- }
- }
/**
* Process the "raw" data to extract vectors of values with {@link NestedPathFinder#find(Object, List)}, wrapping the
* result in {@link StructuredData}
*/
- public static class RawFieldVectorObjectSelector implements VectorObjectSelector
+ public static final class RawFieldVectorObjectSelector implements VectorObjectSelector
{
private final VectorObjectSelector baseSelector;
private final List<NestedPathPart> parts;
@@ -1642,7 +928,10 @@ public class NestedFieldVirtualColumn implements VirtualColumn
}
}
- public static class FieldDimensionSelector extends BaseSingleValueDimensionSelector
+ /**
+ * Create a {@link DimensionSelector} for a nested field on top of a {@link ColumnValueSelector}
+ */
+ public static final class FieldDimensionSelector extends BaseSingleValueDimensionSelector
{
private final ColumnValueSelector<?> valueSelector;
@@ -1665,6 +954,13 @@ public class NestedFieldVirtualColumn implements VirtualColumn
if (val == null || val instanceof String) {
return (String) val;
}
+ if (val instanceof Object[]) {
+ Object[] arrayVal = (Object[]) val;
+ if (arrayVal.length == 1) {
+ return String.valueOf(arrayVal[0]);
+ }
+ return null;
+ }
return String.valueOf(val);
}
}
@@ -1677,7 +973,7 @@ public class NestedFieldVirtualColumn implements VirtualColumn
* This is used as a fall-back when making a selector and the underlying column is NOT a
* {@link NestedDataComplexColumn}, whose field {@link DimensionSelector} natively implement this behavior.
*/
- private static class BestEffortCastingValueSelector implements DimensionSelector
+ private static final class BestEffortCastingValueSelector implements DimensionSelector
{
private final DimensionSelector baseSelector;
@@ -1803,4 +1099,403 @@ public class NestedFieldVirtualColumn implements VirtualColumn
return baseSelector.idLookup();
}
}
+
+ /**
+ * {@link BaseLongVectorValueSelector} for selecting a specific element of an array type column
+ * {@link VectorObjectSelector}
+ */
+ private static final class ArrayElementLongVectorValueSelector extends BaseLongVectorValueSelector
+ {
+ private final long[] longs;
+ private final boolean[] nulls;
+ private final VectorObjectSelector arraySelector;
+ private final int elementNumber;
+ private int id;
+
+ public ArrayElementLongVectorValueSelector(
+ ReadableVectorOffset offset,
+ VectorObjectSelector arraySelector,
+ int elementNumber
+ )
+ {
+ super(offset);
+ this.arraySelector = arraySelector;
+ this.elementNumber = elementNumber;
+ longs = new long[offset.getMaxVectorSize()];
+ nulls = new boolean[offset.getMaxVectorSize()];
+ id = ReadableVectorInspector.NULL_ID;
+ }
+
+ private void computeNumbers()
+ {
+ if (offset.getId() != id) {
+ final Object[] maybeArrays = arraySelector.getObjectVector();
+ for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
+ Object maybeArray = maybeArrays[i];
+ if (maybeArray instanceof Object[]) {
+ Object[] anArray = (Object[]) maybeArray;
+ if (elementNumber < anArray.length) {
+ if (anArray[elementNumber] instanceof Number) {
+ Number n = (Number) anArray[elementNumber];
+ longs[i] = n.longValue();
+ nulls[i] = false;
+ } else {
+ Double d = anArray[elementNumber] instanceof String
+ ? Doubles.tryParse((String) anArray[elementNumber])
+ : null;
+ if (d != null) {
+ longs[i] = d.longValue();
+ nulls[i] = false;
+ } else {
+ longs[i] = 0L;
+ nulls[i] = true;
+ }
+ }
+ } else {
+ nullElement(i);
+ }
+ } else {
+ // not an array?
+ nullElement(i);
+ }
+ }
+ id = offset.getId();
+ }
+ }
+
+ private void nullElement(int i)
+ {
+ longs[i] = 0L;
+ nulls[i] = true;
+ }
+
+ @Override
+ public long[] getLongVector()
+ {
+ if (offset.getId() != id) {
+ computeNumbers();
+ }
+ return longs;
+ }
+
+ @Nullable
+ @Override
+ public boolean[] getNullVector()
+ {
+ if (offset.getId() != id) {
+ computeNumbers();
+ }
+ return nulls;
+ }
+ }
+
+ /**
+ * {@link BaseFloatVectorValueSelector} for selecting a specific element of an array type column
+ * {@link VectorObjectSelector}
+ */
+ private static final class ArrayElementFloatVectorValueSelector extends BaseFloatVectorValueSelector
+ {
+ private final float[] floats;
+ private final boolean[] nulls;
+ private final VectorObjectSelector arraySelector;
+ private final int elementNumber;
+ private int id;
+
+ public ArrayElementFloatVectorValueSelector(
+ ReadableVectorOffset offset,
+ VectorObjectSelector arraySelector,
+ int elementNumber
+ )
+ {
+ super(offset);
+ this.arraySelector = arraySelector;
+ this.elementNumber = elementNumber;
+ floats = new float[offset.getMaxVectorSize()];
+ nulls = new boolean[offset.getMaxVectorSize()];
+ id = ReadableVectorInspector.NULL_ID;
+ }
+
+ private void computeNumbers()
+ {
+ if (offset.getId() != id) {
+ final Object[] maybeArrays = arraySelector.getObjectVector();
+ for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
+ Object maybeArray = maybeArrays[i];
+ if (maybeArray instanceof Object[]) {
+ Object[] anArray = (Object[]) maybeArray;
+ if (elementNumber < anArray.length) {
+ if (anArray[elementNumber] instanceof Number) {
+ Number n = (Number) anArray[elementNumber];
+ floats[i] = n.floatValue();
+ nulls[i] = false;
+ } else {
+ Double d = anArray[elementNumber] instanceof String
+ ? Doubles.tryParse((String) anArray[elementNumber])
+ : null;
+ if (d != null) {
+ floats[i] = d.floatValue();
+ nulls[i] = false;
+ } else {
+ nullElement(i);
+ }
+ }
+ } else {
+ nullElement(i);
+ }
+ } else {
+ // not an array?
+ nullElement(i);
+ }
+ }
+ id = offset.getId();
+ }
+ }
+
+ private void nullElement(int i)
+ {
+ floats[i] = 0f;
+ nulls[i] = true;
+ }
+
+ @Override
+ public float[] getFloatVector()
+ {
+ if (offset.getId() != id) {
+ computeNumbers();
+ }
+ return floats;
+ }
+
+ @Override
+ public boolean[] getNullVector()
+ {
+ if (offset.getId() != id) {
+ computeNumbers();
+ }
+ return nulls;
+ }
+ }
+
+ /**
+ * {@link BaseDoubleVectorValueSelector} for selecting a specific element of an array type column
+ * {@link VectorObjectSelector}
+ */
+ private static final class ArrayElementDoubleVectorValueSelector extends BaseDoubleVectorValueSelector
+ {
+ private final double[] doubles;
+ private final boolean[] nulls;
+ private final VectorObjectSelector arraySelector;
+ private final int elementNumber;
+ private int id;
+
+ public ArrayElementDoubleVectorValueSelector(
+ ReadableVectorOffset offset,
+ VectorObjectSelector arraySelector,
+ int elementNumber
+ )
+ {
+ super(offset);
+ this.arraySelector = arraySelector;
+ this.elementNumber = elementNumber;
+ doubles = new double[offset.getMaxVectorSize()];
+ nulls = new boolean[offset.getMaxVectorSize()];
+ id = ReadableVectorInspector.NULL_ID;
+ }
+
+ private void computeNumbers()
+ {
+ if (offset.getId() != id) {
+ final Object[] maybeArrays = arraySelector.getObjectVector();
+ for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
+ Object maybeArray = maybeArrays[i];
+ if (maybeArray instanceof Object[]) {
+ Object[] anArray = (Object[]) maybeArray;
+ if (elementNumber < anArray.length) {
+ if (anArray[elementNumber] instanceof Number) {
+ Number n = (Number) anArray[elementNumber];
+ doubles[i] = n.doubleValue();
+ nulls[i] = false;
+ } else {
+ Double d = anArray[elementNumber] instanceof String
+ ? Doubles.tryParse((String) anArray[elementNumber])
+ : null;
+ if (d != null) {
+ doubles[i] = d;
+ nulls[i] = false;
+ } else {
+ nullElement(i);
+ }
+ }
+ } else {
+ nullElement(i);
+ }
+ } else {
+ // not an array?
+ nullElement(i);
+ }
+ }
+ id = offset.getId();
+ }
+ }
+
+ private void nullElement(int i)
+ {
+ doubles[i] = 0.0;
+ nulls[i] = true;
+ }
+
+ @Override
+ public double[] getDoubleVector()
+ {
+ if (offset.getId() != id) {
+ computeNumbers();
+ }
+ return doubles;
+ }
+
+ @Override
+ public boolean[] getNullVector()
+ {
+ if (offset.getId() != id) {
+ computeNumbers();
+ }
+ return nulls;
+ }
+ }
+
+ /**
+ * {@link VectorObjectSelector} for selecting a specific element of an array type column
+ * {@link VectorObjectSelector}
+ */
+ private static final class ArrayElementVectorObjectSelector implements VectorObjectSelector
+ {
+ private final Object[] elements;
+ private final VectorObjectSelector arraySelector;
+ private final ReadableVectorOffset offset;
+ private final int elementNumber;
+ private final ExpressionType elementType;
+ private final ExpressionType castTo;
+ private int id;
+
+ public ArrayElementVectorObjectSelector(
+ VectorObjectSelector arraySelector,
+ ReadableVectorOffset offset,
+ int elementNumber,
+ ExpressionType elementType,
+ ExpressionType castTo
+ )
+ {
+ this.arraySelector = arraySelector;
+ this.offset = offset;
+ this.elementNumber = elementNumber;
+ this.elementType = elementType;
+ this.castTo = castTo;
+ elements = new Object[arraySelector.getMaxVectorSize()];
+ id = ReadableVectorInspector.NULL_ID;
+ }
+
+ @Override
+ public Object[] getObjectVector()
+ {
+ if (offset.getId() != id) {
+ final Object[] delegate = arraySelector.getObjectVector();
+ for (int i = 0; i < arraySelector.getCurrentVectorSize(); i++) {
+ Object maybeArray = delegate[i];
+ if (maybeArray instanceof Object[]) {
+ Object[] anArray = (Object[]) maybeArray;
+ if (elementNumber < anArray.length) {
+ elements[i] = ExprEval.ofType(elementType, anArray[elementNumber]).castTo(castTo).value();
+ } else {
+ elements[i] = null;
+ }
+ } else {
+ elements[i] = null;
+ }
+ }
+ id = offset.getId();
+ }
+ return elements;
+ }
+
+ @Override
+ public int getMaxVectorSize()
+ {
+ return arraySelector.getMaxVectorSize();
+ }
+
+ @Override
+ public int getCurrentVectorSize()
+ {
+ return arraySelector.getCurrentVectorSize();
+ }
+ }
+
+ /**
+ * {@link ColumnValueSelector} for selecting a specific element of an array type column {@link ColumnValueSelector}
+ */
+ private static final class ArrayElementColumnValueSelector implements ColumnValueSelector<Object>
+ {
+ private final ColumnValueSelector<?> arraySelector;
+ private final int elementNumber;
+
+ public ArrayElementColumnValueSelector(ColumnValueSelector<?> arraySelector, int elementNumber)
+ {
+ this.arraySelector = arraySelector;
+ this.elementNumber = elementNumber;
+ }
+
+ @Override
+ public boolean isNull()
+ {
+ Object o = getObject();
+ return !(o instanceof Number);
+ }
+
+ @Override
+ public long getLong()
+ {
+ Object o = getObject();
+ return o instanceof Number ? ((Number) o).longValue() : 0L;
+ }
+
+ @Override
+ public float getFloat()
+ {
+ Object o = getObject();
+ return o instanceof Number ? ((Number) o).floatValue() : 0f;
+ }
+
+ @Override
+ public double getDouble()
+ {
+ Object o = getObject();
+ return o instanceof Number ? ((Number) o).doubleValue() : 0.0;
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+ arraySelector.inspectRuntimeShape(inspector);
+ }
+
+ @Nullable
+ @Override
+ public Object getObject()
+ {
+ Object o = arraySelector.getObject();
+ if (o instanceof Object[]) {
+ Object[] array = (Object[]) o;
+ if (elementNumber < array.length) {
+ return array[elementNumber];
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Class<?> classOfObject()
+ {
+ return Object.class;
+ }
+ }
}
diff --git a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
index b18b8cff3ea..e05f98c4540 100644
--- a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
+++ b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
@@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import junitparams.converters.Nullable;
import org.apache.druid.collections.SerializablePair;
-import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.column.TypeStrategies;
import org.apache.druid.segment.column.TypeStrategiesTest;
@@ -445,91 +444,73 @@ public class EvalTest extends InitializedNullHandlingTest
@Test
public void testStringArrayToScalarStringBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.STRING)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<STRING>] to [STRING]", t.getMessage());
+ ExprEval cast = ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.STRING);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.STRING, cast.type());
}
@Test
public void testStringArrayToScalarLongBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.LONG)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<STRING>] to [LONG]", t.getMessage());
+ ExprEval cast = ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.LONG);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.LONG, cast.type());
}
@Test
public void testStringArrayToScalarDoubleBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.DOUBLE)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<STRING>] to [DOUBLE]", t.getMessage());
+ ExprEval cast = ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.DOUBLE);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.DOUBLE, cast.type());
}
@Test
public void testLongArrayToScalarStringBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.STRING)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<LONG>] to [STRING]", t.getMessage());
+ ExprEval cast = ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.STRING);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.STRING, cast.type());
}
@Test
public void testLongArrayToScalarLongBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.LONG)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<LONG>] to [LONG]", t.getMessage());
+ ExprEval cast = ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.LONG);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.LONG, cast.type());
}
@Test
public void testLongArrayToScalarDoubleBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.DOUBLE)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<LONG>] to [DOUBLE]", t.getMessage());
+ ExprEval cast = ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.DOUBLE);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.DOUBLE, cast.type());
}
@Test
public void testDoubleArrayToScalarStringBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.STRING)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<DOUBLE>] to [STRING]", t.getMessage());
+ ExprEval cast = ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.STRING);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.STRING, cast.type());
}
@Test
public void testDoubleArrayToScalarLongBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.LONG)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<DOUBLE>] to [LONG]", t.getMessage());
+ ExprEval cast = ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.LONG);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.LONG, cast.type());
}
@Test
public void testDoubleArrayToScalarDoubleBadCast()
{
- Throwable t = Assert.assertThrows(
- IAE.class,
- () -> ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.DOUBLE)
- );
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<DOUBLE>] to [DOUBLE]", t.getMessage());
+ ExprEval cast = ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.DOUBLE);
+ Assert.assertNull(cast.value());
+ Assert.assertEquals(ExpressionType.DOUBLE, cast.type());
}
@Test
diff --git a/processing/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java b/processing/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java
index e624ed04e50..528d4844bf1 100644
--- a/processing/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java
+++ b/processing/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java
@@ -704,15 +704,14 @@ public class ExprEvalTest extends InitializedNullHandlingTest
cast = eval.castTo(ExpressionTypeFactory.getInstance().ofArray(ExpressionType.NESTED_DATA));
Assert.assertArrayEquals(new Object[]{"1", "2", "foo", null, "3.3"}, (Object[]) cast.value());
- ExprEval<?> finalEval = eval;
- Throwable t = Assert.assertThrows(IAE.class, () -> finalEval.castTo(ExpressionType.LONG));
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<STRING>] to [LONG]", t.getMessage());
+ cast = eval.castTo(ExpressionType.LONG);
+ Assert.assertNull(cast.value());
- t = Assert.assertThrows(IAE.class, () -> finalEval.castTo(ExpressionType.DOUBLE));
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<STRING>] to [DOUBLE]", t.getMessage());
+ cast = eval.castTo(ExpressionType.DOUBLE);
+ Assert.assertNull(cast.value());
- t = Assert.assertThrows(IAE.class, () -> finalEval.castTo(ExpressionType.STRING));
- Assert.assertEquals("Invalid type, cannot cast [ARRAY<STRING>] to [STRING]", t.getMessage());
+ cast = eval.castTo(ExpressionType.STRING);
+ Assert.assertNull(cast.value());
eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[]{1234L});
diff --git a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
index 1ef233237ed..027b076d2de 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
@@ -780,7 +780,7 @@ public class NestedDataScanQueryTest extends InitializedNullHandlingTest
Assert.assertEquals(1, resultsRealtime.size());
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
Assert.assertEquals(
- "[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}, c=null, v=[]}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, 0, [...]
+ "[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}, c=null, v=[]}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c], d=[]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, 0, [...]
resultsSegments.get(0).getEvents().toString()
);
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
diff --git a/processing/src/test/java/org/apache/druid/segment/TestHelper.java b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
index 23589ca065c..737ce8abfbd 100644
--- a/processing/src/test/java/org/apache/druid/segment/TestHelper.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
@@ -395,7 +395,9 @@ public class TestHelper
final Object actualValue = actualMap.get(key);
- if (expectedValue != null && expectedValue.getClass().isArray()) {
+ if ((expectedValue != null && actualValue == null) || (expectedValue == null && actualValue != null)) {
+ Assert.assertEquals(StringUtils.format("%s: key[%s]", msg, key), expectedValue, actualValue);
+ } else if (expectedValue != null && expectedValue.getClass().isArray()) {
Assert.assertArrayEquals((Object[]) expectedValue, (Object[]) actualValue);
} else if (expectedValue instanceof Float || expectedValue instanceof Double) {
Assert.assertEquals(
diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
index 2ed144e5552..818de015448 100644
--- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionSelectorsTest.java
@@ -22,11 +22,22 @@ package org.apache.druid.segment.virtual;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import org.apache.druid.common.guava.SettableSupplier;
+import org.apache.druid.data.input.InputRow;
+import org.apache.druid.data.input.ListBasedInputRow;
import org.apache.druid.data.input.MapBasedInputRow;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.DoubleDimensionSchema;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.frame.FrameType;
+import org.apache.druid.frame.segment.FrameSegment;
+import org.apache.druid.frame.testutil.FrameTestUtil;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
@@ -45,12 +56,21 @@ import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorFactory;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.DimensionSelector;
+import org.apache.druid.segment.IncrementalIndexSegment;
+import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexCursorFactory;
+import org.apache.druid.segment.QueryableIndexSegment;
+import org.apache.druid.segment.RowAdapters;
+import org.apache.druid.segment.RowBasedSegment;
+import org.apache.druid.segment.Segment;
+import org.apache.druid.segment.SimpleAscendingOffset;
import org.apache.druid.segment.TestObjectColumnSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.data.Offset;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
import org.apache.druid.segment.generator.SegmentGenerator;
@@ -61,13 +81,18 @@ import org.apache.druid.segment.incremental.IndexSizeExceededException;
import org.apache.druid.segment.incremental.OnheapIncrementalIndex;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.apache.druid.utils.CloseableUtils;
+import org.joda.time.DateTime;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -132,6 +157,10 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
});
}
+ @Rule
+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+
@Test
public void test_single_value_string_bindings()
{
@@ -707,6 +736,385 @@ public class ExpressionSelectorsTest extends InitializedNullHandlingTest
}
}
+ @Test
+ public void testCastSelectors() throws IOException
+ {
+ final RowSignature rowSignature = RowSignature.builder()
+ .add("string", ColumnType.STRING)
+ .add("multiString", ColumnType.STRING)
+ .add("long", ColumnType.LONG)
+ .add("double", ColumnType.DOUBLE)
+ .add("stringArray", ColumnType.STRING_ARRAY)
+ .add("longArray", ColumnType.LONG_ARRAY)
+ .add("doubleArray", ColumnType.DOUBLE_ARRAY)
+ .build();
+ final DateTime start = DateTimes.nowUtc();
+ List<InputRow> rows = List.of(
+ new ListBasedInputRow(
+ rowSignature,
+ start,
+ rowSignature.getColumnNames(),
+ List.of(
+ "a",
+ List.of("a1", "a2"),
+ 1L,
+ 1.1,
+ new Object[]{"a1", "a2"},
+ new Object[]{1L, 1L},
+ new Object[]{1.1, 1.1}
+ )
+ ),
+ new ListBasedInputRow(
+ rowSignature,
+ start.plusMinutes(1),
+ rowSignature.getColumnNames(),
+ List.of(
+ "b",
+ List.of("b1"),
+ 2L,
+ 2.2,
+ new Object[]{"2.2"},
+ new Object[]{2L},
+ new Object[]{2.2}
+ )
+ ),
+ new ListBasedInputRow(
+ rowSignature,
+ start.plusMinutes(2),
+ rowSignature.getColumnNames(),
+ Lists.newArrayList(
+ null,
+ List.of(),
+ null,
+ null,
+ null,
+ null,
+ null
+ )
+ ),
+ new ListBasedInputRow(
+ rowSignature,
+ start.plusMinutes(3),
+ rowSignature.getColumnNames(),
+ List.of(
+ "4",
+ Lists.newArrayList(null, null, "4.4"),
+ 4L,
+ 4.4,
+ new Object[0],
+ new Object[0],
+ new Object[0]
+ )
+ )
+ );
+
+ final IncrementalIndexSchema schema =
+ IncrementalIndexSchema.builder()
+ .withDimensionsSpec(
+ DimensionsSpec.builder()
+ .setDimensions(
+ List.of(
+ new StringDimensionSchema("string"),
+ new StringDimensionSchema("multiString"),
+ new LongDimensionSchema("long"),
+ new DoubleDimensionSchema("double")
+ )
+ )
+ .useSchemaDiscovery(true)
+ .build()
+ )
+ .build();
+
+ IndexBuilder bob = IndexBuilder.create()
+ .schema(schema)
+ .tmpDir(temporaryFolder.newFolder())
+ .rows(rows);
+
+ try (final Closer closer = Closer.create()) {
+ final List<Segment> segments = List.of(
+ new IncrementalIndexSegment(bob.buildIncrementalIndex(), SegmentId.dummy("test")),
+ new QueryableIndexSegment(bob.buildMMappedIndex(), SegmentId.dummy("test")),
+ new RowBasedSegment<>(Sequences.simple(rows), RowAdapters.standardRow(), RowSignature.empty()),
+ new RowBasedSegment<>(Sequences.simple(rows), RowAdapters.standardRow(), rowSignature),
+ FrameTestUtil.cursorFactoryToFrameSegment(
+ new QueryableIndexCursorFactory(bob.buildMMappedIndex()),
+ FrameType.ROW_BASED
+ ),
+ FrameTestUtil.cursorFactoryToFrameSegment(
+ new QueryableIndexCursorFactory(bob.buildMMappedIndex()),
+ FrameType.COLUMNAR
+ )
+ );
+
+ for (Segment segment : segments) {
+ final CursorFactory cursorFactory = segment.as(CursorFactory.class);
+ Assert.assertNotNull(cursorFactory);
+ final CursorHolder holder = closer.register(cursorFactory.makeCursorHolder(CursorBuildSpec.FULL_SCAN));
+
+ final Cursor cursor = holder.asCursor();
+ final Offset offset = new SimpleAscendingOffset(rows.size());
+
+ ColumnValueSelector baseStringSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("string");
+ ColumnValueSelector baseMultiStringSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("multiString");
+ ColumnValueSelector baseLongSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("long");
+ ColumnValueSelector baseDoubleSelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("double");
+ ColumnValueSelector baseStringArraySelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("stringArray");
+ ColumnValueSelector baseLongArraySelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("longArray");
+ ColumnValueSelector baseDoubleArraySelector = cursor.getColumnSelectorFactory().makeColumnValueSelector("doubleArray");
+
+ ColumnValueSelector stringSelectorToLong = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseStringSelector,
+ ColumnType.STRING,
+ ColumnType.LONG
+ );
+ ColumnValueSelector stringSelectorToDouble = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseStringSelector,
+ ColumnType.STRING,
+ ColumnType.DOUBLE
+ );
+ ColumnValueSelector stringToArraySelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseStringSelector,
+ ColumnType.STRING,
+ ColumnType.STRING_ARRAY
+ );
+
+ ColumnValueSelector multiStringToStringSelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseMultiStringSelector,
+ ColumnType.STRING,
+ ColumnType.STRING
+ );
+
+ ColumnValueSelector multiStringToStringArraySelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseMultiStringSelector,
+ ColumnType.STRING,
+ ColumnType.STRING_ARRAY
+ );
+
+ ColumnValueSelector longToStringSelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseLongSelector,
+ ColumnType.LONG,
+ ColumnType.STRING
+ );
+
+ ColumnValueSelector longToDoubleArraySelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseLongSelector,
+ ColumnType.LONG,
+ ColumnType.DOUBLE_ARRAY
+ );
+
+ ColumnValueSelector doubleToStringSelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseDoubleSelector,
+ ColumnType.DOUBLE,
+ ColumnType.STRING
+ );
+
+ ColumnValueSelector doubleToLongArraySelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseDoubleSelector,
+ ColumnType.DOUBLE,
+ ColumnType.LONG_ARRAY
+ );
+
+ ColumnValueSelector stringArrayToStringSelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseStringArraySelector,
+ ColumnType.STRING_ARRAY,
+ ColumnType.STRING
+ );
+
+ ColumnValueSelector stringArrayToLongArraySelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseStringArraySelector,
+ ColumnType.STRING_ARRAY,
+ ColumnType.LONG_ARRAY
+ );
+
+ ColumnValueSelector stringArrayToDoubleSelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseStringArraySelector,
+ ColumnType.STRING_ARRAY,
+ ColumnType.DOUBLE
+ );
+
+ ColumnValueSelector longArrayToStringSelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseLongArraySelector,
+ ColumnType.LONG_ARRAY,
+ ColumnType.STRING
+ );
+
+ ColumnValueSelector longArrayToDoubleArraySelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseLongArraySelector,
+ ColumnType.LONG_ARRAY,
+ ColumnType.DOUBLE_ARRAY
+ );
+
+ ColumnValueSelector doubleArrayToStringArraySelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseDoubleArraySelector,
+ ColumnType.DOUBLE_ARRAY,
+ ColumnType.STRING_ARRAY
+ );
+
+ ColumnValueSelector doubleArrayToLongSelector = ExpressionSelectors.castColumnValueSelector(
+ offset::getOffset,
+ baseDoubleArraySelector,
+ ColumnType.DOUBLE_ARRAY,
+ ColumnType.LONG
+ );
+
+ // first row
+ // "a"
+ Assert.assertNull(stringSelectorToLong.getObject());
+ Assert.assertNull(stringSelectorToDouble.getObject());
+ Assert.assertArrayEquals(new Object[]{"a"}, (Object[]) stringToArraySelector.getObject());
+
+ // ["a1", "a2"]
+ Assert.assertNull(multiStringToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{"a1", "a2"}, (Object[]) multiStringToStringArraySelector.getObject());
+
+ // 1
+ Assert.assertEquals("1", longToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{1.0}, (Object[]) longToDoubleArraySelector.getObject());
+
+ // 1.1
+ Assert.assertEquals("1.1", doubleToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{1L}, (Object[]) doubleToLongArraySelector.getObject());
+
+ // ["a1", "a2"]
+ Assert.assertNull(stringArrayToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{null, null}, (Object[]) stringArrayToLongArraySelector.getObject());
+ Assert.assertNull(stringArrayToDoubleSelector.getObject());
+
+ // [1, 1]
+ Assert.assertNull(longArrayToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{1.0, 1.0}, (Object[]) longArrayToDoubleArraySelector.getObject());
+
+ // [1.1, 1.1]
+ Assert.assertArrayEquals(new Object[]{"1.1", "1.1"}, (Object[]) doubleArrayToStringArraySelector.getObject());
+ Assert.assertNull(doubleArrayToLongSelector.getObject());
+
+ cursor.advance();
+ offset.increment();
+
+ // first row
+ // "b"
+ Assert.assertNull(stringSelectorToLong.getObject());
+ Assert.assertNull(stringSelectorToDouble.getObject());
+ Assert.assertArrayEquals(new Object[]{"b"}, (Object[]) stringToArraySelector.getObject());
+
+ // ["b1"]
+ Assert.assertEquals("b1", multiStringToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{"b1"}, (Object[]) multiStringToStringArraySelector.getObject());
+
+ // 2
+ Assert.assertEquals("2", longToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{2.0}, (Object[]) longToDoubleArraySelector.getObject());
+
+ // 2.2
+ Assert.assertEquals("2.2", doubleToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{2L}, (Object[]) doubleToLongArraySelector.getObject());
+
+ // ["2.2"]
+ Assert.assertEquals("2.2", stringArrayToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{2L}, (Object[]) stringArrayToLongArraySelector.getObject());
+ Assert.assertEquals(2.2, stringArrayToDoubleSelector.getObject());
+
+ // [2]
+ Assert.assertEquals("2", longArrayToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{2.0}, (Object[]) longArrayToDoubleArraySelector.getObject());
+
+ // [2.2]
+ Assert.assertArrayEquals(new Object[]{"2.2"}, (Object[]) doubleArrayToStringArraySelector.getObject());
+ Assert.assertEquals(2L, doubleArrayToLongSelector.getObject());
+
+ cursor.advance();
+ offset.increment();
+
+
+ // third row
+ // null
+ Assert.assertNull(stringSelectorToLong.getObject());
+ Assert.assertNull(stringSelectorToDouble.getObject());
+ Assert.assertNull(stringToArraySelector.getObject());
+
+ // []
+ Assert.assertNull(multiStringToStringSelector.getObject());
+ if (segment instanceof IncrementalIndexSegment || segment instanceof QueryableIndexSegment || segment instanceof FrameSegment) {
+ Assert.assertNull(multiStringToStringSelector.getObject());
+ } else {
+ // this one is kind of weird, the row based segment selector does not convert the empty list into null like
+ // the others do
+ Assert.assertArrayEquals(new Object[0], (Object[]) multiStringToStringArraySelector.getObject());
+ }
+
+ // null
+ Assert.assertNull(longToStringSelector.getObject());
+ Assert.assertNull(longToDoubleArraySelector.getObject());
+
+ // null
+ Assert.assertNull(doubleToStringSelector.getObject());
+ Assert.assertNull(doubleToLongArraySelector.getObject());
+
+ // null
+ Assert.assertNull(stringArrayToStringSelector.getObject());
+ Assert.assertNull(stringArrayToLongArraySelector.getObject());
+ Assert.assertNull(stringArrayToDoubleSelector.getObject());
+
+ // null
+ Assert.assertNull(longArrayToStringSelector.getObject());
+ Assert.assertNull(longArrayToDoubleArraySelector.getObject());
+
+ // null
+ Assert.assertNull(doubleArrayToStringArraySelector.getObject());
+ Assert.assertNull(doubleArrayToLongSelector.getObject());
+
+ cursor.advance();
+ offset.increment();
+
+ // fourth row
+ // "4"
+ Assert.assertEquals(4L, stringSelectorToLong.getObject());
+ Assert.assertEquals(4.0, stringSelectorToDouble.getObject());
+ Assert.assertArrayEquals(new Object[]{"4"}, (Object[]) stringToArraySelector.getObject());
+
+ // [null, null, 4.4]
+ Assert.assertNull(multiStringToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{null, null, "4.4"}, (Object[]) multiStringToStringArraySelector.getObject());
+
+ // 4
+ Assert.assertEquals("4", longToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{4.0}, (Object[]) longToDoubleArraySelector.getObject());
+
+ // 4.4
+ Assert.assertEquals("4.4", doubleToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[]{4L}, (Object[]) doubleToLongArraySelector.getObject());
+
+ // []
+ Assert.assertNull(stringArrayToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[0], (Object[]) stringArrayToLongArraySelector.getObject());
+ Assert.assertNull(stringArrayToDoubleSelector.getObject());
+
+ // []
+ Assert.assertNull(longArrayToStringSelector.getObject());
+ Assert.assertArrayEquals(new Object[0], (Object[]) longArrayToDoubleArraySelector.getObject());
+
+ // []
+ Assert.assertArrayEquals(new Object[0], (Object[]) doubleArrayToStringArraySelector.getObject());
+ Assert.assertNull(doubleArrayToLongSelector.getObject());
+ }
+ }
+ }
+
private static DimensionSelector dimensionSelectorFromSupplier(
final Supplier<String> supplier
)
diff --git a/processing/src/test/resources/nested-all-types-test-data.json b/processing/src/test/resources/nested-all-types-test-data.json
index 5572c72799b..e8d4a4ac26e 100644
--- a/processing/src/test/resources/nested-all-types-test-data.json
+++ b/processing/src/test/resources/nested-all-types-test-data.json
@@ -1,4 +1,4 @@
-{"timestamp": "2023-01-01T00:00:00", "str":"a", "long":1, "double":1.0, "bool": true, "variant": 1, "variantNumeric": 1, "variantEmptyObj":1, "variantEmtpyArray":1, "variantWithArrays": 1, "obj":{"a": 100, "b": {"x": "a", "y": 1.1, "z": [1, 2, 3, 4]}, "c": 100, "v": []}, "complexObj":{"x": 1234, "y": [{"l": ["a", "b", "c"], "m": "a", "n": 1},{"l": ["a", "b", "c"], "m": "a", "n": 1}], "z": {"a": [1.1, 2.2, 3.3], "b": true}}, "arrayString": ["a", "b"], [...]
+{"timestamp": "2023-01-01T00:00:00", "str":"a", "long":1, "double":1.0, "bool": true, "variant": 1, "variantNumeric": 1, "variantEmptyObj":1, "variantEmtpyArray":1, "variantWithArrays": 1, "obj":{"a": 100, "b": {"x": "a", "y": 1.1, "z": [1, 2, 3, 4]}, "c": [100], "v": []}, "complexObj":{"x": 1234, "y": [{"l": ["a", "b", "c"], "m": "a", "n": 1},{"l": ["a", "b", "c"], "m": "a", "n": 1}], "z": {"a": [1.1, 2.2, 3.3], "b": true}}, "arrayString": ["a", "b"], [...]
{"timestamp": "2023-01-01T00:00:00", "str":"", "long":2, "bool": false, "variant": "b", "variantNumeric": 1.1, "variantEmptyObj":"b", "variantEmtpyArray":2, "variantWithArrays": "b", "obj":{"a": 200, "b": {"x": "b", "y": 1.1, "z": [2, 4, 6]}, "c": ["a", "b"], "v": []}, "complexObj":{"x": 10, "y": [{"l": ["b", "b", "c"], "m": "b", "n": 2}, [1, 2, 3]], "z": {"a": [5.5], "b": false}}, "arrayString": ["a", "b", "c"] [...]
{"timestamp": "2023-01-01T00:00:00", "str":"null", "long":3, "double":2.0, "variant": 3.0, "variantNumeric": 1.0, "variantEmptyObj":3.3, "variantEmtpyArray":3, "variantWithArrays": 3.0, "obj":{"a": 300}, "complexObj":{"x": 4.4, "y": [{"l": [], "m": 100, "n": 3},{"l": ["a"]}, {"l": ["b"], "n": []}], "z": {"a": [], "b": true}}, "arrayString": ["b", "c"], [...]
{"timestamp": "2023-01-01T00:00:00", "str":"b", "long":4, "double":3.3, "bool": true, "variant": "1", "variantEmptyObj":{}, "variantEmtpyArray":4, "variantWithArrays": "1", "obj":{"a": 400, "b": {"x": "d", "y": 1.1, "z": [3, 4]}, "c": {"a": 1},"v": []}, "complexObj":{"x": 1234, "z": {"a": [1.1, 2.2, 3.3], "b": true}}, "arrayString": ["d", "e"], [...]
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
index 7aab9d34837..afae66b062d 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
@@ -1502,7 +1502,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
outprint(col);
outprint("F");
} else if (col instanceof Object[]) {
- printArray(array);
+ printArray((Object[]) col);
} else if (col instanceof List) {
printList((List<?>) col);
} else {
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
index 8c525918ab8..42367523646 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
@@ -6075,7 +6075,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
"1",
"1",
"1",
- "{\"a\":100,\"b\":{\"x\":\"a\",\"y\":1.1,\"z\":[1,2,3,4]},\"c\":100,\"v\":[]}",
+ "{\"a\":100,\"b\":{\"x\":\"a\",\"y\":1.1,\"z\":[1,2,3,4]},\"c\":[100],\"v\":[]}",
"{\"x\":1234,\"y\":[{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1},{\"l\":[\"a\",\"b\",\"c\"],\"m\":\"a\",\"n\":1}],\"z\":{\"a\":[1.1,2.2,3.3],\"b\":true}}",
"[\"a\",\"b\"]",
"[\"a\",\"b\"]",
@@ -6801,44 +6801,93 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
@Test
public void testJsonValueNestedEmptyArray()
{
- // The data set has empty arrays, however MSQ returns nulls. The root cause of the issue is the incorrect
- // capabilities returned by NestedFieldVirtualColumn when planning which causes MSQ to treat the nested path
- // as STRING, even though it is an array.
- msqIncompatible();
- // test for regression
skipVectorize();
- testQuery(
- "SELECT json_value(cObj, '$.z.d') FROM druid.all_auto",
- ImmutableList.of(
- Druids.newScanQueryBuilder()
- .dataSource(DATA_SOURCE_ALL)
- .intervals(querySegmentSpec(Filtration.eternity()))
- .columns("v0")
- .columnTypes(ColumnType.STRING)
- .virtualColumns(
- new NestedFieldVirtualColumn(
- "cObj",
- "$.z.d",
- "v0",
- ColumnType.STRING
+ testBuilder()
+ .sql("SELECT json_value(cObj, '$.z.d') FROM druid.all_auto")
+ .queryContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .expectedQueries(
+ ImmutableList.of(
+ Druids.newScanQueryBuilder()
+ .dataSource(DATA_SOURCE_ALL)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .columns("v0")
+ .columnTypes(ColumnType.STRING)
+ .virtualColumns(
+ new NestedFieldVirtualColumn(
+ "cObj",
+ "$.z.d",
+ "v0",
+ ColumnType.STRING
+ )
)
- )
- .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
- .build()
- ),
- ImmutableList.of(
- new Object[]{"[]"},
- new Object[]{"[]"},
- new Object[]{"[]"},
- new Object[]{"[]"},
- new Object[]{"[]"},
- new Object[]{"[]"},
- new Object[]{"[]"}
- ),
- RowSignature.builder()
- .add("EXPR$0", ColumnType.STRING)
- .build()
- );
+ .context(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+ .build()
+ )
+ )
+ .expectedResults(
+ ImmutableList.of(
+ new Object[]{null},
+ new Object[]{null},
+ new Object[]{null},
+ new Object[]{null},
+ new Object[]{null},
+ new Object[]{null},
+ new Object[]{null}
+ )
+ )
+ .expectedSignature(
+ RowSignature.builder()
+ .add("EXPR$0", ColumnType.STRING)
+ .build()
+ )
+ .run();
+ }
+
+ @Test
+ public void testJsonValueNestedEmptyArrayReturning()
+ {
+ skipVectorize();
+ testBuilder()
+ .sql("SELECT json_value(cObj, '$.z.d' returning varchar array) FROM druid.all_auto")
+ .queryContext(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .expectedQueries(
+ ImmutableList.of(
+ Druids.newScanQueryBuilder()
+ .dataSource(DATA_SOURCE_ALL)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .columns("v0")
+ .columnTypes(ColumnType.STRING_ARRAY)
+ .virtualColumns(
+ new NestedFieldVirtualColumn(
+ "cObj",
+ "$.z.d",
+ "v0",
+ ColumnType.STRING_ARRAY
+ )
+ )
+ .context(QUERY_CONTEXT_NO_STRINGIFY_ARRAY)
+ .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+ .build()
+ )
+ )
+ .expectedResults(
+ ImmutableList.of(
+ new Object[]{Collections.emptyList()},
+ new Object[]{Collections.emptyList()},
+ new Object[]{Collections.emptyList()},
+ new Object[]{Collections.emptyList()},
+ new Object[]{Collections.emptyList()},
+ new Object[]{Collections.emptyList()},
+ new Object[]{Collections.emptyList()}
+ )
+ )
+ .expectedSignature(
+ RowSignature.builder()
+ .add("EXPR$0", ColumnType.STRING_ARRAY)
+ .build()
+ )
+ .run();
}
@Test
@@ -7265,7 +7314,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
public void testSumPathWithArrays()
{
/*
- "obj":{... "c": 100, ...}
+ "obj":{... "c": [100], ...}
"obj":{... "c": ["a", "b"], ...}
"obj":{...}
"obj":{... "c": {"a": 1}, ...},
@@ -7300,7 +7349,7 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
public void testCountPathWithArrays()
{
/*
- "obj":{... "c": 100, ...}
+ "obj":{... "c": [100], ...}
"obj":{... "c": ["a", "b"], ...}
"obj":{...}
"obj":{... "c": {"a": 1}, ...},
@@ -7342,11 +7391,53 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
);
}
+ @Test
+ public void testCountPathWithArraysReturningNumber()
+ {
+ /*
+ "obj":{... "c": [100], ...}
+ "obj":{... "c": ["a", "b"], ...}
+ "obj":{...}
+ "obj":{... "c": {"a": 1}, ...},
+ "obj":{... "c": "hello", ...},
+ "obj":{... "c": 12.3, ...},
+ "obj":{... "c": null, ...},
+ */
+ testQuery(
+ "SELECT "
+ + "COUNT(JSON_VALUE(obj, '$.c' RETURNING DOUBLE)) "
+ + "FROM druid.all_auto",
+ ImmutableList.of(
+ Druids.newTimeseriesQueryBuilder()
+ .dataSource(DATA_SOURCE_ALL)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .granularity(Granularities.ALL)
+ .virtualColumns(new NestedFieldVirtualColumn("obj", "$.c", "v0", ColumnType.DOUBLE))
+ .aggregators(
+ aggregators(
+ new FilteredAggregatorFactory(
+ new CountAggregatorFactory("a0"),
+ not(isNull("v0"))
+ )
+ )
+ )
+ .context(QUERY_CONTEXT_DEFAULT)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{2L}
+ ),
+ RowSignature.builder()
+ .add("EXPR$0", ColumnType.LONG)
+ .build()
+ );
+ }
+
@Test
public void testCountPathWithArraysReturning()
{
/*
- "obj":{... "c": 100, ...}
+ "obj":{... "c": [100], ...}
"obj":{... "c": ["a", "b"], ...}
"obj":{...}
"obj":{... "c": {"a": 1}, ...},
@@ -7383,4 +7474,173 @@ public class CalciteNestedDataQueryTest extends BaseCalciteQueryTest
.build()
);
}
+
+ @Test
+ public void testSumPathWithArraysRealtime()
+ {
+ /*
+ "obj":{... "c": [100], ...}
+ "obj":{... "c": ["a", "b"], ...}
+ "obj":{...}
+ "obj":{... "c": {"a": 1}, ...},
+ "obj":{... "c": "hello", ...},
+ "obj":{... "c": 12.3, ...},
+ "obj":{... "c": null, ...},
+ */
+ skipVectorize();
+ testQuery(
+ "SELECT "
+ + "SUM(JSON_VALUE(obj, '$.c')) "
+ + "FROM druid.all_auto_realtime",
+ ImmutableList.of(
+ Druids.newTimeseriesQueryBuilder()
+ .dataSource(DATA_SOURCE_ALL_REALTIME)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .granularity(Granularities.ALL)
+ .virtualColumns(new NestedFieldVirtualColumn("obj", "$.c", "v0", ColumnType.DOUBLE))
+ .aggregators(aggregators(new DoubleSumAggregatorFactory("a0", "v0")))
+ .context(QUERY_CONTEXT_DEFAULT)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{112.3d}
+ ),
+ RowSignature.builder()
+ .add("EXPR$0", ColumnType.DOUBLE)
+ .build()
+ );
+ }
+
+ @Test
+ public void testCountPathWithArraysRealtime()
+ {
+ /*
+ "obj":{... "c": [100], ...}
+ "obj":{... "c": ["a", "b"], ...}
+ "obj":{...}
+ "obj":{... "c": {"a": 1}, ...},
+ "obj":{... "c": "hello", ...},
+ "obj":{... "c": 12.3, ...},
+ "obj":{... "c": null, ...},
+ */
+ // capturing existing behavior... the count should be 4 if it was counting all non-null primitive values, but that
+ // would mean that the virtual column would need to plan as ARRAY<STRING> expected type instead of STRING
+ // ... you might notice there are actually 5 non-null obj.c values, however json_value only returns primitive
+ // values, so the object row is rightfully skipped
+ skipVectorize();
+ testQuery(
+ "SELECT "
+ + "COUNT(JSON_VALUE(obj, '$.c')) "
+ + "FROM druid.all_auto_realtime",
+ ImmutableList.of(
+ Druids.newTimeseriesQueryBuilder()
+ .dataSource(DATA_SOURCE_ALL_REALTIME)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .granularity(Granularities.ALL)
+ .virtualColumns(new NestedFieldVirtualColumn("obj", "$.c", "v0", ColumnType.STRING))
+ .aggregators(
+ aggregators(
+ new FilteredAggregatorFactory(
+ new CountAggregatorFactory("a0"),
+ not(isNull("v0"))
+ )
+ )
+ )
+ .context(QUERY_CONTEXT_DEFAULT)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{3L}
+ ),
+ RowSignature.builder()
+ .add("EXPR$0", ColumnType.LONG)
+ .build()
+ );
+ }
+
+ @Test
+ public void testCountPathWithArraysReturningNumberRealtime()
+ {
+ /*
+ "obj":{... "c": [100], ...}
+ "obj":{... "c": ["a", "b"], ...}
+ "obj":{...}
+ "obj":{... "c": {"a": 1}, ...},
+ "obj":{... "c": "hello", ...},
+ "obj":{... "c": 12.3, ...},
+ "obj":{... "c": null, ...},
+ */
+ skipVectorize();
+ testQuery(
+ "SELECT "
+ + "COUNT(JSON_VALUE(obj, '$.c' RETURNING DOUBLE)) "
+ + "FROM druid.all_auto_realtime",
+ ImmutableList.of(
+ Druids.newTimeseriesQueryBuilder()
+ .dataSource(DATA_SOURCE_ALL_REALTIME)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .granularity(Granularities.ALL)
+ .virtualColumns(new NestedFieldVirtualColumn("obj", "$.c", "v0", ColumnType.DOUBLE))
+ .aggregators(
+ aggregators(
+ new FilteredAggregatorFactory(
+ new CountAggregatorFactory("a0"),
+ not(isNull("v0"))
+ )
+ )
+ )
+ .context(QUERY_CONTEXT_DEFAULT)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{2L}
+ ),
+ RowSignature.builder()
+ .add("EXPR$0", ColumnType.LONG)
+ .build()
+ );
+ }
+
+ @Test
+ public void testCountPathWithArraysReturningRealtime()
+ {
+ /*
+ "obj":{... "c": [100], ...}
+ "obj":{... "c": ["a", "b"], ...}
+ "obj":{...}
+ "obj":{... "c": {"a": 1}, ...},
+ "obj":{... "c": "hello", ...},
+ "obj":{... "c": 12.3, ...},
+ "obj":{... "c": null, ...},
+ */
+ skipVectorize();
+ testQuery(
+ "SELECT "
+ + "COUNT(JSON_VALUE(obj, '$.c' RETURNING VARCHAR ARRAY)) "
+ + "FROM druid.all_auto_realtime",
+ ImmutableList.of(
+ Druids.newTimeseriesQueryBuilder()
+ .dataSource(DATA_SOURCE_ALL_REALTIME)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .granularity(Granularities.ALL)
+ .virtualColumns(new NestedFieldVirtualColumn("obj", "$.c", "v0", ColumnType.STRING_ARRAY))
+ .aggregators(
+ aggregators(
+ new FilteredAggregatorFactory(
+ new CountAggregatorFactory("a0"),
+ not(isNull("v0"))
+ )
+ )
+ )
+ .context(QUERY_CONTEXT_DEFAULT)
+ .build()
+ ),
+ ImmutableList.of(
+ new Object[]{4L}
+ ),
+ RowSignature.builder()
+ .add("EXPR$0", ColumnType.LONG)
+ .build()
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org