You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by jo...@apache.org on 2020/10/08 06:57:02 UTC
[druid] branch 0.20.0 updated: vectorized group by support for
nullable numeric columns (#10441) (#10490)
This is an automated email from the ASF dual-hosted git repository.
jonwei pushed a commit to branch 0.20.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/0.20.0 by this push:
new b3b2538 vectorized group by support for nullable numeric columns (#10441) (#10490)
b3b2538 is described below
commit b3b25386479f821248584fb87c7903dc8d99cc9e
Author: Jonathan Wei <jo...@users.noreply.github.com>
AuthorDate: Wed Oct 7 23:56:48 2020 -0700
vectorized group by support for nullable numeric columns (#10441) (#10490)
* vectorized group by support for numeric null columns
* revert unintended change
* adjust
* review stuffs
Co-authored-by: Clint Wylie <cw...@apache.org>
---
.../VectorValueMatcherColumnProcessorFactory.java | 18 ++-
.../epinephelinae/RowBasedGrouperHelper.java | 67 +++++---
.../GroupByVectorColumnProcessorFactory.java | 51 ++++++-
.../NullableDoubleGroupByVectorColumnSelector.java | 82 ++++++++++
.../NullableFloatGroupByVectorColumnSelector.java | 82 ++++++++++
.../NullableLongGroupByVectorColumnSelector.java | 82 ++++++++++
.../epinephelinae/vector/VectorGroupByEngine.java | 2 +-
.../druid/segment/DimensionHandlerUtils.java | 5 +
.../segment/VectorColumnProcessorFactory.java | 17 ++-
...ctorValueMatcherColumnProcessorFactoryTest.java | 67 +++++++-
.../query/groupby/GroupByQueryRunnerTest.java | 169 ++++++++++++++++++++-
.../virtual/VectorizedVirtualColumnTest.java | 13 --
.../apache/druid/sql/calcite/CalciteQueryTest.java | 44 ------
13 files changed, 590 insertions(+), 109 deletions(-)
diff --git a/processing/src/main/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactory.java b/processing/src/main/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactory.java
index 5ca511f..b2083cc 100644
--- a/processing/src/main/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactory.java
@@ -20,6 +20,7 @@
package org.apache.druid.query.filter.vector;
import org.apache.druid.segment.VectorColumnProcessorFactory;
+import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector;
import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector;
import org.apache.druid.segment.vector.VectorValueSelector;
@@ -40,6 +41,7 @@ public class VectorValueMatcherColumnProcessorFactory implements VectorColumnPro
@Override
public VectorValueMatcherFactory makeSingleValueDimensionProcessor(
+ final ColumnCapabilities capabilities,
final SingleValueDimensionVectorSelector selector
)
{
@@ -48,6 +50,7 @@ public class VectorValueMatcherColumnProcessorFactory implements VectorColumnPro
@Override
public VectorValueMatcherFactory makeMultiValueDimensionProcessor(
+ final ColumnCapabilities capabilities,
final MultiValueDimensionVectorSelector selector
)
{
@@ -55,19 +58,28 @@ public class VectorValueMatcherColumnProcessorFactory implements VectorColumnPro
}
@Override
- public VectorValueMatcherFactory makeFloatProcessor(final VectorValueSelector selector)
+ public VectorValueMatcherFactory makeFloatProcessor(
+ final ColumnCapabilities capabilities,
+ final VectorValueSelector selector
+ )
{
return new FloatVectorValueMatcher(selector);
}
@Override
- public VectorValueMatcherFactory makeDoubleProcessor(final VectorValueSelector selector)
+ public VectorValueMatcherFactory makeDoubleProcessor(
+ final ColumnCapabilities capabilities,
+ final VectorValueSelector selector
+ )
{
return new DoubleVectorValueMatcher(selector);
}
@Override
- public VectorValueMatcherFactory makeLongProcessor(final VectorValueSelector selector)
+ public VectorValueMatcherFactory makeLongProcessor(
+ final ColumnCapabilities capabilities,
+ final VectorValueSelector selector
+ )
{
return new LongVectorValueMatcher(selector);
}
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
index d21e609..c099eed 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java
@@ -736,7 +736,6 @@ public class RowBasedGrouperHelper
{
private final boolean includeTimestamp;
private final boolean sortByDimsFirst;
- private final int dimCount;
private final long maxDictionarySize;
private final DefaultLimitSpec limitSpec;
private final List<DimensionSpec> dimensions;
@@ -756,7 +755,6 @@ public class RowBasedGrouperHelper
this.includeTimestamp = includeTimestamp;
this.sortByDimsFirst = sortByDimsFirst;
this.dimensions = dimensions;
- this.dimCount = dimensions.size();
this.maxDictionarySize = maxDictionarySize;
this.limitSpec = limitSpec;
this.aggregatorFactories = aggregatorFactories;
@@ -807,7 +805,7 @@ public class RowBasedGrouperHelper
if (includeTimestamp) {
if (sortByDimsFirst) {
return (entry1, entry2) -> {
- final int cmp = compareDimsInRows(entry1.getKey(), entry2.getKey(), 1);
+ final int cmp = compareDimsInRows(entry1.getKey(), entry2.getKey(), valueTypes, 1);
if (cmp != 0) {
return cmp;
}
@@ -825,22 +823,23 @@ public class RowBasedGrouperHelper
return timeCompare;
}
- return compareDimsInRows(entry1.getKey(), entry2.getKey(), 1);
+ return compareDimsInRows(entry1.getKey(), entry2.getKey(), valueTypes, 1);
};
}
} else {
- return (entry1, entry2) -> compareDimsInRows(entry1.getKey(), entry2.getKey(), 0);
+ return (entry1, entry2) -> compareDimsInRows(entry1.getKey(), entry2.getKey(), valueTypes, 0);
}
}
private Comparator<Grouper.Entry<RowBasedKey>> objectComparatorWithAggs()
{
// use the actual sort order from the limitspec if pushing down to merge partial results correctly
+ final int dimCount = dimensions.size();
final List<Boolean> needsReverses = new ArrayList<>();
final List<Boolean> aggFlags = new ArrayList<>();
- final List<Boolean> isNumericField = new ArrayList<>();
final List<StringComparator> comparators = new ArrayList<>();
final List<Integer> fieldIndices = new ArrayList<>();
+ final List<ValueType> fieldValueTypes = new ArrayList<>();
final Set<Integer> orderByIndices = new HashSet<>();
for (OrderByColumnSpec orderSpec : limitSpec.getColumns()) {
@@ -852,7 +851,7 @@ public class RowBasedGrouperHelper
needsReverses.add(needsReverse);
aggFlags.add(false);
final ValueType type = dimensions.get(dimIndex).getOutputType();
- isNumericField.add(ValueType.isNumeric(type));
+ fieldValueTypes.add(type);
comparators.add(orderSpec.getDimensionComparator());
} else {
int aggIndex = OrderByColumnSpec.getAggIndexForOrderBy(orderSpec, Arrays.asList(aggregatorFactories));
@@ -860,7 +859,7 @@ public class RowBasedGrouperHelper
fieldIndices.add(aggIndex);
needsReverses.add(needsReverse);
aggFlags.add(true);
- isNumericField.add(aggregatorFactories[aggIndex].getType().isNumeric());
+ fieldValueTypes.add(aggregatorFactories[aggIndex].getType());
comparators.add(orderSpec.getDimensionComparator());
}
}
@@ -871,9 +870,9 @@ public class RowBasedGrouperHelper
fieldIndices.add(i);
aggFlags.add(false);
needsReverses.add(false);
- boolean isNumeric = ValueType.isNumeric(dimensions.get(i).getOutputType());
- isNumericField.add(isNumeric);
- if (isNumeric) {
+ ValueType type = dimensions.get(i).getOutputType();
+ fieldValueTypes.add(type);
+ if (type.isNumeric()) {
comparators.add(StringComparators.NUMERIC);
} else {
comparators.add(StringComparators.LEXICOGRAPHIC);
@@ -891,7 +890,7 @@ public class RowBasedGrouperHelper
needsReverses,
aggFlags,
fieldIndices,
- isNumericField,
+ fieldValueTypes,
comparators
);
if (cmp != 0) {
@@ -918,7 +917,7 @@ public class RowBasedGrouperHelper
needsReverses,
aggFlags,
fieldIndices,
- isNumericField,
+ fieldValueTypes,
comparators
);
};
@@ -931,19 +930,33 @@ public class RowBasedGrouperHelper
needsReverses,
aggFlags,
fieldIndices,
- isNumericField,
+ fieldValueTypes,
comparators
);
}
}
- private static int compareDimsInRows(RowBasedKey key1, RowBasedKey key2, int dimStart)
+ private static int compareDimsInRows(RowBasedKey key1, RowBasedKey key2, final List<ValueType> fieldTypes, int dimStart)
{
for (int i = dimStart; i < key1.getKey().length; i++) {
- final int cmp = Comparators.<Comparable>naturalNullsFirst().compare(
- (Comparable) key1.getKey()[i],
- (Comparable) key2.getKey()[i]
- );
+ final int cmp;
+ // sometimes doubles can become floats making the round trip from serde, make sure to coerce them both
+ // to double
+ // timestamp is not present in fieldTypes since it only includes the dimensions. sort of hacky, but if timestamp
+ // is included, dimstart will be 1, so subtract from 'i' to get correct index
+ if (ValueType.DOUBLE == fieldTypes.get(i - dimStart)) {
+ Object lhs = key1.getKey()[i];
+ Object rhs = key2.getKey()[i];
+ cmp = Comparators.<Comparable>naturalNullsFirst().compare(
+ lhs != null ? ((Number) lhs).doubleValue() : null,
+ rhs != null ? ((Number) rhs).doubleValue() : null
+ );
+ } else {
+ cmp = Comparators.<Comparable>naturalNullsFirst().compare(
+ (Comparable) key1.getKey()[i],
+ (Comparable) key2.getKey()[i]
+ );
+ }
if (cmp != 0) {
return cmp;
}
@@ -959,7 +972,7 @@ public class RowBasedGrouperHelper
final List<Boolean> needsReverses,
final List<Boolean> aggFlags,
final List<Integer> fieldIndices,
- final List<Boolean> isNumericField,
+ final List<ValueType> fieldTypes,
final List<StringComparator> comparators
)
{
@@ -990,9 +1003,19 @@ public class RowBasedGrouperHelper
final StringComparator comparator = comparators.get(i);
- if (isNumericField.get(i) && comparator.equals(StringComparators.NUMERIC)) {
+ final ValueType fieldType = fieldTypes.get(i);
+ if (fieldType.isNumeric() && comparator.equals(StringComparators.NUMERIC)) {
// use natural comparison
- cmp = Comparators.<Comparable>naturalNullsFirst().compare(lhs, rhs);
+ if (ValueType.DOUBLE == fieldType) {
+ // sometimes doubles can become floats making the round trip from serde, make sure to coerce them both
+ // to double
+ cmp = Comparators.<Comparable>naturalNullsFirst().compare(
+ lhs != null ? ((Number) lhs).doubleValue() : null,
+ rhs != null ? ((Number) rhs).doubleValue() : null
+ );
+ } else {
+ cmp = Comparators.<Comparable>naturalNullsFirst().compare(lhs, rhs);
+ }
} else {
cmp = comparator.compare(
DimensionHandlerUtils.convertObjectToString(lhs),
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/GroupByVectorColumnProcessorFactory.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/GroupByVectorColumnProcessorFactory.java
index b42d34e..46dec35 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/GroupByVectorColumnProcessorFactory.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/GroupByVectorColumnProcessorFactory.java
@@ -19,7 +19,10 @@
package org.apache.druid.query.groupby.epinephelinae.vector;
+import com.google.common.base.Preconditions;
import org.apache.druid.segment.VectorColumnProcessorFactory;
+import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector;
import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector;
import org.apache.druid.segment.vector.VectorValueSelector;
@@ -39,32 +42,64 @@ public class GroupByVectorColumnProcessorFactory implements VectorColumnProcesso
}
@Override
- public GroupByVectorColumnSelector makeSingleValueDimensionProcessor(final SingleValueDimensionVectorSelector selector)
+ public GroupByVectorColumnSelector makeSingleValueDimensionProcessor(
+ final ColumnCapabilities capabilities,
+ final SingleValueDimensionVectorSelector selector
+ )
{
+ Preconditions.checkArgument(
+ ValueType.STRING == capabilities.getType(),
+ "groupBy dimension processors must be STRING typed"
+ );
return new SingleValueStringGroupByVectorColumnSelector(selector);
}
@Override
- public GroupByVectorColumnSelector makeMultiValueDimensionProcessor(final MultiValueDimensionVectorSelector selector)
+ public GroupByVectorColumnSelector makeMultiValueDimensionProcessor(
+ final ColumnCapabilities capabilities,
+ final MultiValueDimensionVectorSelector selector
+ )
{
+ Preconditions.checkArgument(
+ ValueType.STRING == capabilities.getType(),
+ "groupBy dimension processors must be STRING typed"
+ );
throw new UnsupportedOperationException("Multi-value dimensions not yet implemented for vectorized groupBys");
}
@Override
- public GroupByVectorColumnSelector makeFloatProcessor(final VectorValueSelector selector)
+ public GroupByVectorColumnSelector makeFloatProcessor(
+ final ColumnCapabilities capabilities,
+ final VectorValueSelector selector
+ )
{
- return new FloatGroupByVectorColumnSelector(selector);
+ if (capabilities.hasNulls().isFalse()) {
+ return new FloatGroupByVectorColumnSelector(selector);
+ }
+ return new NullableFloatGroupByVectorColumnSelector(selector);
}
@Override
- public GroupByVectorColumnSelector makeDoubleProcessor(final VectorValueSelector selector)
+ public GroupByVectorColumnSelector makeDoubleProcessor(
+ final ColumnCapabilities capabilities,
+ final VectorValueSelector selector
+ )
{
- return new DoubleGroupByVectorColumnSelector(selector);
+ if (capabilities.hasNulls().isFalse()) {
+ return new DoubleGroupByVectorColumnSelector(selector);
+ }
+ return new NullableDoubleGroupByVectorColumnSelector(selector);
}
@Override
- public GroupByVectorColumnSelector makeLongProcessor(final VectorValueSelector selector)
+ public GroupByVectorColumnSelector makeLongProcessor(
+ final ColumnCapabilities capabilities,
+ final VectorValueSelector selector
+ )
{
- return new LongGroupByVectorColumnSelector(selector);
+ if (capabilities.hasNulls().isFalse()) {
+ return new LongGroupByVectorColumnSelector(selector);
+ }
+ return new NullableLongGroupByVectorColumnSelector(selector);
}
}
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableDoubleGroupByVectorColumnSelector.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableDoubleGroupByVectorColumnSelector.java
new file mode 100644
index 0000000..26a5844
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableDoubleGroupByVectorColumnSelector.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.groupby.epinephelinae.vector;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.query.groupby.ResultRow;
+import org.apache.druid.segment.vector.VectorValueSelector;
+
+public class NullableDoubleGroupByVectorColumnSelector implements GroupByVectorColumnSelector
+{
+ private final VectorValueSelector selector;
+
+ NullableDoubleGroupByVectorColumnSelector(final VectorValueSelector selector)
+ {
+ this.selector = selector;
+ }
+
+ @Override
+ public int getGroupingKeySize()
+ {
+ return Byte.BYTES + Double.BYTES;
+ }
+
+ @Override
+ public void writeKeys(
+ final WritableMemory keySpace,
+ final int keySize,
+ final int keyOffset,
+ final int startRow,
+ final int endRow
+ )
+ {
+ final double[] vector = selector.getDoubleVector();
+ final boolean[] nulls = selector.getNullVector();
+
+ if (nulls != null) {
+ for (int i = startRow, j = keyOffset; i < endRow; i++, j += keySize) {
+ keySpace.putByte(j, nulls[i] ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE);
+ keySpace.putDouble(j + 1, vector[i]);
+ }
+ } else {
+ for (int i = startRow, j = keyOffset; i < endRow; i++, j += keySize) {
+ keySpace.putByte(j, NullHandling.IS_NOT_NULL_BYTE);
+ keySpace.putDouble(j + 1, vector[i]);
+ }
+ }
+ }
+
+ @Override
+ public void writeKeyToResultRow(
+ final Memory keyMemory,
+ final int keyOffset,
+ final ResultRow resultRow,
+ final int resultRowPosition
+ )
+ {
+ if (keyMemory.getByte(keyOffset) == NullHandling.IS_NULL_BYTE) {
+ resultRow.set(resultRowPosition, null);
+ } else {
+ resultRow.set(resultRowPosition, keyMemory.getDouble(keyOffset + 1));
+ }
+ }
+}
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableFloatGroupByVectorColumnSelector.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableFloatGroupByVectorColumnSelector.java
new file mode 100644
index 0000000..a2a0c60
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableFloatGroupByVectorColumnSelector.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.groupby.epinephelinae.vector;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.query.groupby.ResultRow;
+import org.apache.druid.segment.vector.VectorValueSelector;
+
+public class NullableFloatGroupByVectorColumnSelector implements GroupByVectorColumnSelector
+{
+ private final VectorValueSelector selector;
+
+ NullableFloatGroupByVectorColumnSelector(final VectorValueSelector selector)
+ {
+ this.selector = selector;
+ }
+
+ @Override
+ public int getGroupingKeySize()
+ {
+ return Byte.BYTES + Float.BYTES;
+ }
+
+ @Override
+ public void writeKeys(
+ final WritableMemory keySpace,
+ final int keySize,
+ final int keyOffset,
+ final int startRow,
+ final int endRow
+ )
+ {
+ final float[] vector = selector.getFloatVector();
+ final boolean[] nulls = selector.getNullVector();
+
+ if (nulls != null) {
+ for (int i = startRow, j = keyOffset; i < endRow; i++, j += keySize) {
+ keySpace.putByte(j, nulls[i] ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE);
+ keySpace.putFloat(j + 1, vector[i]);
+ }
+ } else {
+ for (int i = startRow, j = keyOffset; i < endRow; i++, j += keySize) {
+ keySpace.putByte(j, NullHandling.IS_NOT_NULL_BYTE);
+ keySpace.putFloat(j + 1, vector[i]);
+ }
+ }
+ }
+
+ @Override
+ public void writeKeyToResultRow(
+ final Memory keyMemory,
+ final int keyOffset,
+ final ResultRow resultRow,
+ final int resultRowPosition
+ )
+ {
+ if (keyMemory.getByte(keyOffset) == NullHandling.IS_NULL_BYTE) {
+ resultRow.set(resultRowPosition, null);
+ } else {
+ resultRow.set(resultRowPosition, keyMemory.getFloat(keyOffset + 1));
+ }
+ }
+}
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableLongGroupByVectorColumnSelector.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableLongGroupByVectorColumnSelector.java
new file mode 100644
index 0000000..ab020e2
--- /dev/null
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/NullableLongGroupByVectorColumnSelector.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.groupby.epinephelinae.vector;
+
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.query.groupby.ResultRow;
+import org.apache.druid.segment.vector.VectorValueSelector;
+
+public class NullableLongGroupByVectorColumnSelector implements GroupByVectorColumnSelector
+{
+ private final VectorValueSelector selector;
+
+ NullableLongGroupByVectorColumnSelector(final VectorValueSelector selector)
+ {
+ this.selector = selector;
+ }
+
+ @Override
+ public int getGroupingKeySize()
+ {
+ return Byte.BYTES + Long.BYTES;
+ }
+
+ @Override
+ public void writeKeys(
+ final WritableMemory keySpace,
+ final int keySize,
+ final int keyOffset,
+ final int startRow,
+ final int endRow
+ )
+ {
+ final long[] vector = selector.getLongVector();
+ final boolean[] nulls = selector.getNullVector();
+
+ if (nulls != null) {
+ for (int i = startRow, j = keyOffset; i < endRow; i++, j += keySize) {
+ keySpace.putByte(j, nulls[i] ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE);
+ keySpace.putLong(j + 1, vector[i]);
+ }
+ } else {
+ for (int i = startRow, j = keyOffset; i < endRow; i++, j += keySize) {
+ keySpace.putByte(j, NullHandling.IS_NOT_NULL_BYTE);
+ keySpace.putLong(j + 1, vector[i]);
+ }
+ }
+ }
+
+ @Override
+ public void writeKeyToResultRow(
+ final Memory keyMemory,
+ final int keyOffset,
+ final ResultRow resultRow,
+ final int resultRowPosition
+ )
+ {
+ if (keyMemory.getByte(keyOffset) == NullHandling.IS_NULL_BYTE) {
+ resultRow.set(resultRowPosition, null);
+ } else {
+ resultRow.set(resultRowPosition, keyMemory.getLong(keyOffset + 1));
+ }
+ }
+}
diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
index f1cf33e..f516ea5 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/vector/VectorGroupByEngine.java
@@ -114,7 +114,7 @@ public class VectorGroupByEngine
columnCapabilities.isDictionaryEncoded().isTrue() &&
columnCapabilities.areDictionaryValuesUnique().isTrue();
}
- return columnCapabilities.hasMultipleValues().isFalse() && columnCapabilities.hasNulls().isFalse();
+ return columnCapabilities.hasMultipleValues().isFalse();
});
}
diff --git a/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java
index f5b7e9f..f4bbba0 100644
--- a/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java
+++ b/processing/src/main/java/org/apache/druid/segment/DimensionHandlerUtils.java
@@ -307,10 +307,12 @@ public final class DimensionHandlerUtils
if (type == ValueType.STRING) {
if (!forceSingleValue && effectiveCapabilites.hasMultipleValues().isMaybeTrue()) {
return strategyFactory.makeMultiValueDimensionProcessor(
+ effectiveCapabilites,
selectorFactory.makeMultiValueDimensionSelector(dimensionSpec)
);
} else {
return strategyFactory.makeSingleValueDimensionProcessor(
+ effectiveCapabilites,
selectorFactory.makeSingleValueDimensionSelector(dimensionSpec)
);
}
@@ -325,14 +327,17 @@ public final class DimensionHandlerUtils
if (type == ValueType.LONG) {
return strategyFactory.makeLongProcessor(
+ effectiveCapabilites,
selectorFactory.makeValueSelector(dimensionSpec.getDimension())
);
} else if (type == ValueType.FLOAT) {
return strategyFactory.makeFloatProcessor(
+ effectiveCapabilites,
selectorFactory.makeValueSelector(dimensionSpec.getDimension())
);
} else if (type == ValueType.DOUBLE) {
return strategyFactory.makeDoubleProcessor(
+ effectiveCapabilites,
selectorFactory.makeValueSelector(dimensionSpec.getDimension())
);
} else {
diff --git a/processing/src/main/java/org/apache/druid/segment/VectorColumnProcessorFactory.java b/processing/src/main/java/org/apache/druid/segment/VectorColumnProcessorFactory.java
index f76f2dc..6ae1557 100644
--- a/processing/src/main/java/org/apache/druid/segment/VectorColumnProcessorFactory.java
+++ b/processing/src/main/java/org/apache/druid/segment/VectorColumnProcessorFactory.java
@@ -19,6 +19,7 @@
package org.apache.druid.segment;
+import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector;
import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector;
import org.apache.druid.segment.vector.VectorValueSelector;
@@ -36,13 +37,19 @@ import org.apache.druid.segment.vector.VectorValueSelector;
*/
public interface VectorColumnProcessorFactory<T>
{
- T makeSingleValueDimensionProcessor(SingleValueDimensionVectorSelector selector);
+ T makeSingleValueDimensionProcessor(
+ ColumnCapabilities capabilities,
+ SingleValueDimensionVectorSelector selector
+ );
- T makeMultiValueDimensionProcessor(MultiValueDimensionVectorSelector selector);
+ T makeMultiValueDimensionProcessor(
+ ColumnCapabilities capabilities,
+ MultiValueDimensionVectorSelector selector
+ );
- T makeFloatProcessor(VectorValueSelector selector);
+ T makeFloatProcessor(ColumnCapabilities capabilities, VectorValueSelector selector);
- T makeDoubleProcessor(VectorValueSelector selector);
+ T makeDoubleProcessor(ColumnCapabilities capabilities, VectorValueSelector selector);
- T makeLongProcessor(VectorValueSelector selector);
+ T makeLongProcessor(ColumnCapabilities capabilities, VectorValueSelector selector);
}
diff --git a/processing/src/test/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactoryTest.java
index ffa0253..9f0e7c2 100644
--- a/processing/src/test/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactoryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/filter/vector/VectorValueMatcherColumnProcessorFactoryTest.java
@@ -21,6 +21,8 @@ package org.apache.druid.query.filter.vector;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.segment.IdLookup;
+import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
+import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector;
import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector;
import org.apache.druid.segment.vector.VectorValueSelector;
@@ -49,7 +51,10 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
public void testFloat()
{
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeFloatProcessor(vectorValueSelector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeFloatProcessor(
+ ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.FLOAT),
+ vectorValueSelector
+ );
Assert.assertTrue(matcherFactory instanceof FloatVectorValueMatcher);
@@ -74,7 +79,10 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
public void testDouble()
{
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeDoubleProcessor(vectorValueSelector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeDoubleProcessor(
+ ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.DOUBLE),
+ vectorValueSelector
+ );
Assert.assertTrue(matcherFactory instanceof DoubleVectorValueMatcher);
@@ -100,7 +108,10 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
public void testLong()
{
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeLongProcessor(vectorValueSelector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeLongProcessor(
+ ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.LONG),
+ vectorValueSelector
+ );
Assert.assertTrue(matcherFactory instanceof LongVectorValueMatcher);
@@ -137,7 +148,15 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
EasyMock.replay(selector, lookup);
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(selector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(
+ new ColumnCapabilitiesImpl().setType(ValueType.STRING)
+ .setHasMultipleValues(false)
+ .setHasBitmapIndexes(true)
+ .setDictionaryValuesUnique(true)
+ .setDictionaryValuesSorted(true)
+ .setDictionaryEncoded(true),
+ selector
+ );
Assert.assertTrue(matcherFactory instanceof SingleValueStringVectorValueMatcher);
@@ -167,7 +186,15 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
EasyMock.replay(selector);
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(selector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(
+ new ColumnCapabilitiesImpl().setType(ValueType.STRING)
+ .setHasMultipleValues(false)
+ .setHasBitmapIndexes(true)
+ .setDictionaryValuesUnique(true)
+ .setDictionaryValuesSorted(true)
+ .setDictionaryEncoded(true),
+ selector
+ );
Assert.assertTrue(matcherFactory instanceof SingleValueStringVectorValueMatcher);
@@ -199,7 +226,15 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
EasyMock.replay(selector);
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(selector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(
+ new ColumnCapabilitiesImpl().setType(ValueType.STRING)
+ .setHasMultipleValues(false)
+ .setHasBitmapIndexes(true)
+ .setDictionaryValuesUnique(true)
+ .setDictionaryValuesSorted(true)
+ .setDictionaryEncoded(true),
+ selector
+ );
Assert.assertTrue(matcherFactory instanceof SingleValueStringVectorValueMatcher);
@@ -234,7 +269,15 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
EasyMock.replay(selector, lookup);
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(selector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeSingleValueDimensionProcessor(
+ new ColumnCapabilitiesImpl().setType(ValueType.STRING)
+ .setHasMultipleValues(false)
+ .setHasBitmapIndexes(true)
+ .setDictionaryValuesUnique(true)
+ .setDictionaryValuesSorted(true)
+ .setDictionaryEncoded(true),
+ selector
+ );
Assert.assertTrue(matcherFactory instanceof SingleValueStringVectorValueMatcher);
@@ -259,7 +302,15 @@ public class VectorValueMatcherColumnProcessorFactoryTest extends InitializedNul
EasyMock.expect(lookup.lookupId(null)).andReturn(0).anyTimes();
EasyMock.replay(selector, lookup);
VectorValueMatcherFactory matcherFactory =
- VectorValueMatcherColumnProcessorFactory.instance().makeMultiValueDimensionProcessor(selector);
+ VectorValueMatcherColumnProcessorFactory.instance().makeMultiValueDimensionProcessor(
+ new ColumnCapabilitiesImpl().setType(ValueType.STRING)
+ .setHasMultipleValues(false)
+ .setHasBitmapIndexes(true)
+ .setDictionaryValuesUnique(true)
+ .setDictionaryValuesSorted(true)
+ .setDictionaryEncoded(true),
+ selector
+ );
Assert.assertTrue(matcherFactory instanceof MultiValueStringVectorValueMatcher);
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
index a4fc11d..751e58a 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java
@@ -190,7 +190,6 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
private static final Closer RESOURCE_CLOSER = Closer.create();
private final QueryRunner<ResultRow> runner;
- private final String runnerName;
private final GroupByQueryRunnerFactory factory;
private final GroupByQueryConfig config;
private final boolean vectorize;
@@ -455,7 +454,7 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
this.config = config;
this.factory = factory;
this.runner = factory.mergeRunners(Execs.directExecutor(), ImmutableList.of(runner));
- this.runnerName = runner.toString();
+ String runnerName = runner.toString();
this.vectorize = vectorize;
}
@@ -10645,15 +10644,175 @@ public class GroupByQueryRunnerTest extends InitializedNullHandlingTest
}
@Test
- public void testGroupByOnVirtualColumn()
+ public void testGroupByOnNullableLong()
+ {
+ if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
+ expectedException.expect(UnsupportedOperationException.class);
+ }
+
+ GroupByQuery query = makeQueryBuilder()
+ .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+ .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+ .setDimensions(
+ new DefaultDimensionSpec("longNumericNull", "nullable", ValueType.LONG)
+ )
+ .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT)
+ .setGranularity(QueryRunnerTestHelper.ALL_GRAN)
+ .setLimit(5)
+ .build();
+
+ List<ResultRow> expectedResults;
+ if (NullHandling.sqlCompatible()) {
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", null, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10L, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20L, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40L, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50L, "rows", 6L)
+ );
+ } else {
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", 0L, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10L, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20L, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40L, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50L, "rows", 6L)
+ );
+ }
+
+ Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
+ TestHelper.assertExpectedObjects(expectedResults, results, "groupBy");
+ }
+
+ @Test
+ public void testGroupByOnNullableDouble()
+ {
+ if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
+ expectedException.expect(UnsupportedOperationException.class);
+ }
+
+ GroupByQuery query = makeQueryBuilder()
+ .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+ .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+ .setDimensions(
+ new DefaultDimensionSpec("doubleNumericNull", "nullable", ValueType.DOUBLE)
+ )
+ .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT)
+ .setGranularity(QueryRunnerTestHelper.ALL_GRAN)
+ .setLimit(5)
+ .build();
+
+ List<ResultRow> expectedResults;
+ if (NullHandling.sqlCompatible()) {
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", null, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50.0, "rows", 6L)
+ );
+ } else {
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", 0.0, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50.0, "rows", 6L)
+ );
+ }
+
+ Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
+ TestHelper.assertExpectedObjects(expectedResults, results, "groupBy");
+ }
+
+ @Test
+ public void testGroupByOnNullableDoubleNoLimitPushdown()
{
if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
expectedException.expect(UnsupportedOperationException.class);
}
- // cannot vectorize due to unknown nulls in numeric column
+ GroupByQuery query = makeQueryBuilder()
+ .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+ .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+ .setDimensions(
+ new DefaultDimensionSpec("doubleNumericNull", "nullable", ValueType.DOUBLE)
+ )
+ .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT)
+ .setGranularity(QueryRunnerTestHelper.ALL_GRAN)
+ .overrideContext(ImmutableMap.of(GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, false))
+ .setLimitSpec(new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("nullable", OrderByColumnSpec.Direction.ASCENDING)), 5))
+ .build();
+
+ List<ResultRow> expectedResults;
if (NullHandling.sqlCompatible()) {
- cannotVectorize();
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", null, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50.0, "rows", 6L)
+ );
+ } else {
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", 0.0, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40.0, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50.0, "rows", 6L)
+ );
+ }
+
+ Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
+ TestHelper.assertExpectedObjects(expectedResults, results, "groupBy");
+ }
+
+ @Test
+ public void testGroupByOnNullableFloat()
+ {
+ if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
+ expectedException.expect(UnsupportedOperationException.class);
+ }
+
+ GroupByQuery query = makeQueryBuilder()
+ .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
+ .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
+ .setDimensions(
+ new DefaultDimensionSpec("floatNumericNull", "nullable", ValueType.FLOAT)
+ )
+ .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT)
+ .setGranularity(QueryRunnerTestHelper.ALL_GRAN)
+ .setLimit(5)
+ .build();
+
+ List<ResultRow> expectedResults;
+ if (NullHandling.sqlCompatible()) {
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", null, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10.0f, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20.0f, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40.0f, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50.0f, "rows", 6L)
+ );
+ } else {
+ expectedResults = Arrays.asList(
+ makeRow(query, "2011-04-01", "nullable", 0.0f, "rows", 6L),
+ makeRow(query, "2011-04-01", "nullable", 10.0f, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 20.0f, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 40.0f, "rows", 2L),
+ makeRow(query, "2011-04-01", "nullable", 50.0f, "rows", 6L)
+ );
+ }
+
+ Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
+ TestHelper.assertExpectedObjects(expectedResults, results, "groupBy");
+ }
+
+ @Test
+ public void testGroupByOnVirtualColumn()
+ {
+ if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
+ expectedException.expect(UnsupportedOperationException.class);
}
GroupByQuery query = makeQueryBuilder()
diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/VectorizedVirtualColumnTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/VectorizedVirtualColumnTest.java
index 4d8d3ef..3aa6eef 100644
--- a/processing/src/test/java/org/apache/druid/segment/virtual/VectorizedVirtualColumnTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/virtual/VectorizedVirtualColumnTest.java
@@ -22,7 +22,6 @@ package org.apache.druid.segment.virtual;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
-import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence;
@@ -182,30 +181,18 @@ public class VectorizedVirtualColumnTest
@Test
public void testGroupByLong()
{
- // vectorized group by does not work for null numeric columns
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testGroupBy(ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.LONG));
}
@Test
public void testGroupByDouble()
{
- // vectorized group by does not work for null numeric columns
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testGroupBy(ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.DOUBLE));
}
@Test
public void testGroupByFloat()
{
- // vectorized group by does not work for null numeric columns
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testGroupBy(ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.FLOAT));
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
index a87c951..5e116c7 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
@@ -5843,10 +5843,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExpressionFilteringAndGrouping() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT\n"
+ " FLOOR(m1 / 2) * 2,\n"
@@ -5893,10 +5889,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExpressionFilteringAndGroupingUsingCastToLong() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT\n"
+ " CAST(m1 AS BIGINT) / 2 * 2,\n"
@@ -5945,10 +5937,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testExpressionFilteringAndGroupingOnStringCastToNumber() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT\n"
+ " FLOOR(CAST(dim1 AS FLOAT) / 2) * 2,\n"
@@ -6693,10 +6681,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testTimeseriesWithTimeFilterOnLongColumnUsingMillisToTimestamp() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT\n"
+ " FLOOR(MILLIS_TO_TIMESTAMP(cnt) TO YEAR),\n"
@@ -9088,10 +9072,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByFloor() throws Exception
{
- // grouping on numeric columns with null values is not yet supported
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT floor(CAST(dim1 AS float)), COUNT(*) FROM druid.foo GROUP BY floor(CAST(dim1 AS float))",
ImmutableList.of(
@@ -9119,10 +9099,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByFloorWithOrderBy() throws Exception
{
- // grouping on numeric columns with null values is not yet supported
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT floor(CAST(dim1 AS float)) AS fl, COUNT(*) FROM druid.foo GROUP BY floor(CAST(dim1 AS float)) ORDER BY fl DESC",
ImmutableList.of(
@@ -9174,10 +9150,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT floor(__time TO year), dim2, COUNT(*)"
+ " FROM druid.foo"
@@ -11416,10 +11388,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testTimeseriesUsingTimeFloorWithTimestampAdd() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT SUM(cnt), gran FROM (\n"
+ " SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'P1M') AS gran,\n"
@@ -11930,10 +11898,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
public void testTimeseriesWithLimitAndOffset() throws Exception
{
// Timeseries cannot handle offsets, so the query morphs into a groupBy.
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT gran, SUM(cnt)\n"
+ "FROM (\n"
@@ -11998,10 +11962,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testGroupByTimeAndOtherDimension() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
testQuery(
"SELECT dim2, gran, SUM(cnt)\n"
+ "FROM (SELECT FLOOR(__time TO MONTH) AS gran, dim2, cnt FROM druid.foo) AS x\n"
@@ -15951,10 +15911,6 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
@Test
public void testRepeatedIdenticalVirtualExpressionGrouping() throws Exception
{
- // cannot vectorize due to unknown nulls in numeric column
- if (NullHandling.sqlCompatible()) {
- cannotVectorize();
- }
final String query = "SELECT \n"
+ "\tCASE dim1 WHEN NULL THEN FALSE ELSE TRUE END AS col_a,\n"
+ "\tCASE dim2 WHEN NULL THEN FALSE ELSE TRUE END AS col_b\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org