You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/09/30 21:45:20 UTC
[incubator-pinot] branch support_array_transform_function_phase1
created (now c5e290b)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch support_array_transform_function_phase1
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at c5e290b Adding array transform functions: array_average, array_max, array_min, array_sum
This branch includes the following new commits:
new c5e290b Adding array transform functions: array_average, array_max, array_min, array_sum
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Adding array transform functions:
array_average, array_max, array_min, array_sum
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch support_array_transform_function_phase1
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit c5e290be29335dd763ccee11f019546247e5d8f2
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Wed Sep 30 14:44:28 2020 -0700
Adding array transform functions: array_average, array_max, array_min, array_sum
---
.../common/function/TransformFunctionType.java | 4 +
.../function/ArrayAverageTransformFunction.java | 127 +++++++++++++++
.../function/ArrayMaxTransformFunction.java | 179 +++++++++++++++++++++
.../function/ArrayMinTransformFunction.java | 178 ++++++++++++++++++++
.../function/ArraySumTransformFunction.java | 148 +++++++++++++++++
.../function/TransformFunctionFactory.java | 6 +
.../ArrayAverageTransformFunctionTest.java | 49 ++++++
.../function/ArrayBaseTransformFunctionTest.java | 97 +++++++++++
.../function/ArrayLengthTransformFunctionTest.java | 38 ++---
.../function/ArrayMaxTransformFunctionTest.java | 49 ++++++
.../function/ArrayMinTransformFunctionTest.java | 49 ++++++
.../function/ArraySumTransformFunctionTest.java | 57 +++++++
12 files changed, 957 insertions(+), 24 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
index 1347f1d..0dea6d2 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/function/TransformFunctionType.java
@@ -53,6 +53,10 @@ public enum TransformFunctionType {
DATETIMECONVERT("dateTimeConvert"),
DATETRUNC("dateTrunc"),
ARRAYLENGTH("arrayLength"),
+ ARRAY_AVERAGE("array_average"),
+ ARRAY_MIN("array_min"),
+ ARRAY_MAX("array_max"),
+ ARRAY_SUM("array_sum"),
VALUEIN("valueIn"),
MAPVALUE("mapValue"),
INIDSET("inIdSet"),
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java
new file mode 100644
index 0000000..e0610ed
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.plan.DocIdSetPlanNode;
+
+
+/**
+ * The ArrayAverageTransformFunction class implements array_average function for multi-valued columns
+ *
+ * Sample queries:
+ * SELECT COUNT(*) FROM table WHERE array_average(mvColumn) > 2
+ * SELECT COUNT(*) FROM table GROUP BY array_average(mvColumn)
+ * SELECT SUM(array_average(mvColumn)) FROM table
+ */
+public class ArrayAverageTransformFunction extends BaseTransformFunction {
+ public static final String FUNCTION_NAME = "array_average";
+
+ private double[] _results;
+ private TransformFunction _argument;
+
+ @Override
+ public String getName() {
+ return FUNCTION_NAME;
+ }
+
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
+ // Check that there is only 1 argument
+ if (arguments.size() != 1) {
+ throw new IllegalArgumentException("Exactly 1 argument is required for ARRAY_AVERAGE transform function");
+ }
+
+ // Check that the argument is a multi-valued column or transform function
+ TransformFunction firstArgument = arguments.get(0);
+ if (firstArgument instanceof LiteralTransformFunction || firstArgument.getResultMetadata().isSingleValue()) {
+ throw new IllegalArgumentException(
+ "The argument of ARRAY_AVERAGE transform function must be a multi-valued column or a transform function");
+ }
+ if (!firstArgument.getResultMetadata().getDataType().isNumeric()) {
+ throw new IllegalArgumentException(
+ "The argument of ARRAY_AVERAGE transform function must be numeric");
+ }
+ _argument = firstArgument;
+ }
+
+ @Override
+ public TransformResultMetadata getResultMetadata() {
+ return DOUBLE_SV_NO_DICTIONARY_METADATA;
+ }
+
+ @Override
+ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
+ if (_results == null) {
+ _results = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+
+ int numDocs = projectionBlock.getNumDocs();
+ double sumRes;
+ switch (_argument.getResultMetadata().getDataType()) {
+ case INT:
+ int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock);
+ for (int i = 0; i < numDocs; i++) {
+ sumRes = 0;
+ for (int j = 0; j < intValuesMV[i].length; j++) {
+ sumRes += intValuesMV[i][j];
+ }
+ _results[i] = sumRes / intValuesMV[i].length;
+ }
+ break;
+ case LONG:
+ long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
+ for (int i = 0; i < numDocs; i++) {
+ sumRes = 0;
+ for (int j = 0; j < longValuesMV[i].length; j++) {
+ sumRes += longValuesMV[i][j];
+ }
+ _results[i] = sumRes / longValuesMV[i].length;
+ }
+ break;
+ case FLOAT:
+ float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock);
+ for (int i = 0; i < numDocs; i++) {
+ sumRes = 0;
+ for (int j = 0; j < floatValuesMV[i].length; j++) {
+ sumRes += floatValuesMV[i][j];
+ }
+ _results[i] = sumRes / floatValuesMV[i].length;
+ }
+ break;
+ case DOUBLE:
+ double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
+ for (int i = 0; i < numDocs; i++) {
+ sumRes = 0;
+ for (int j = 0; j < doubleValuesMV[i].length; j++) {
+ sumRes += doubleValuesMV[i][j];
+ }
+ _results[i] = sumRes / doubleValuesMV[i].length;
+ }
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ return _results;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java
new file mode 100644
index 0000000..f877244
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.plan.DocIdSetPlanNode;
+import org.apache.pinot.core.util.ArrayCopyUtils;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+/**
+ * The ArrayMaxTransformFunction class implements array_max function for multi-valued columns
+ *
+ * Sample queries:
+ * SELECT COUNT(*) FROM table WHERE array_max(mvColumn) > 2
+ * SELECT COUNT(*) FROM table GROUP BY array_max(mvColumn)
+ * SELECT SUM(array_max(mvColumn)) FROM table
+ */
+public class ArrayMaxTransformFunction extends BaseTransformFunction {
+ public static final String FUNCTION_NAME = "array_max";
+
+ private int[] _intValuesSV;
+ private long[] _longValuesSV;
+ private float[] _floatValuesSV;
+ private double[] _doubleValuesSV;
+ private String[] _stringValuesSV;
+ private TransformFunction _argument;
+ private TransformResultMetadata _resultMetadata;
+
+ @Override
+ public String getName() {
+ return FUNCTION_NAME;
+ }
+
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
+ // Check that there is only 1 argument
+ if (arguments.size() != 1) {
+ throw new IllegalArgumentException("Exactly 1 argument is required for ARRAY_MAX transform function");
+ }
+
+ // Check that the argument is a multi-valued column or transform function
+ TransformFunction firstArgument = arguments.get(0);
+ if (firstArgument instanceof LiteralTransformFunction || firstArgument.getResultMetadata().isSingleValue()) {
+ throw new IllegalArgumentException(
+ "The argument of ARRAY_MAX transform function must be a multi-valued column or a transform function");
+ }
+ _resultMetadata = new TransformResultMetadata(firstArgument.getResultMetadata().getDataType(), true, false);
+ _argument = firstArgument;
+ }
+
+ @Override
+ public TransformResultMetadata getResultMetadata() {
+ return _resultMetadata;
+ }
+
+ @Override
+ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.INT) {
+ return super.transformToIntValuesSV(projectionBlock);
+ }
+ if (_intValuesSV == null) {
+ _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ int maxRes = Integer.MIN_VALUE;
+ for (int j = 0; j < intValuesMV[i].length; j++) {
+ maxRes = Math.max(maxRes, intValuesMV[i][j]);
+ }
+ _intValuesSV[i] = maxRes;
+ }
+ return _intValuesSV;
+ }
+
+ @Override
+ public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.LONG) {
+ return super.transformToLongValuesSV(projectionBlock);
+ }
+ if (_longValuesSV == null) {
+ _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ long maxRes = Long.MIN_VALUE;
+ for (int j = 0; j < longValuesMV[i].length; j++) {
+ maxRes = Math.max(maxRes, longValuesMV[i][j]);
+ }
+ _longValuesSV[i] = maxRes;
+ }
+ return _longValuesSV;
+ }
+
+ @Override
+ public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.FLOAT) {
+ return super.transformToFloatValuesSV(projectionBlock);
+ }
+ if (_floatValuesSV == null) {
+ _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ float maxRes = Float.NEGATIVE_INFINITY;
+ for (int j = 0; j < floatValuesMV[i].length; j++) {
+ maxRes = Math.max(maxRes, floatValuesMV[i][j]);
+ }
+ _floatValuesSV[i] = maxRes;
+ }
+ return _floatValuesSV;
+ }
+
+ @Override
+ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.DOUBLE) {
+ return super.transformToDoubleValuesSV(projectionBlock);
+ }
+ if (_doubleValuesSV == null) {
+ _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ double maxRes = Double.NEGATIVE_INFINITY;
+ for (int j = 0; j < doubleValuesMV[i].length; j++) {
+ maxRes = Math.max(maxRes, doubleValuesMV[i][j]);
+ }
+ _doubleValuesSV[i] = maxRes;
+ }
+ return _doubleValuesSV;
+ }
+
+ @Override
+ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.STRING) {
+ return super.transformToStringValuesSV(projectionBlock);
+ }
+ if (_stringValuesSV == null) {
+ _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ String[][] stringValuesMV = _argument.transformToStringValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ String maxRes = null;
+ for (int j = 0; j < stringValuesMV[i].length; j++) {
+ if (StringUtils.compare(maxRes, stringValuesMV[i][j]) < 0) {
+ maxRes = stringValuesMV[i][j];
+ }
+ }
+ _stringValuesSV[i] = maxRes;
+ }
+ return _stringValuesSV;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java
new file mode 100644
index 0000000..86abf33
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.plan.DocIdSetPlanNode;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+/**
+ * The ArrayMinTransformFunction class implements array_min function for multi-valued columns
+ *
+ * Sample queries:
+ * SELECT COUNT(*) FROM table WHERE array_min(mvColumn) > 2
+ * SELECT COUNT(*) FROM table GROUP BY array_min(mvColumn)
+ * SELECT SUM(array_min(mvColumn)) FROM table
+ */
+public class ArrayMinTransformFunction extends BaseTransformFunction {
+ public static final String FUNCTION_NAME = "array_min";
+
+ private int[] _intValuesSV;
+ private long[] _longValuesSV;
+ private float[] _floatValuesSV;
+ private double[] _doubleValuesSV;
+ private String[] _stringValuesSV;
+ private TransformFunction _argument;
+ private TransformResultMetadata _resultMetadata;
+
+ @Override
+ public String getName() {
+ return FUNCTION_NAME;
+ }
+
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
+ // Check that there is only 1 argument
+ if (arguments.size() != 1) {
+ throw new IllegalArgumentException("Exactly 1 argument is required for ARRAY_MIN transform function");
+ }
+
+ // Check that the argument is a multi-valued column or transform function
+ TransformFunction firstArgument = arguments.get(0);
+ if (firstArgument instanceof LiteralTransformFunction || firstArgument.getResultMetadata().isSingleValue()) {
+ throw new IllegalArgumentException(
+ "The argument of ARRAY_MIN transform function must be a multi-valued column or a transform function");
+ }
+ _resultMetadata = new TransformResultMetadata(firstArgument.getResultMetadata().getDataType(), true, false);
+ _argument = firstArgument;
+ }
+
+ @Override
+ public TransformResultMetadata getResultMetadata() {
+ return _resultMetadata;
+ }
+
+ @Override
+ public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.INT) {
+ return super.transformToIntValuesSV(projectionBlock);
+ }
+ if (_intValuesSV == null) {
+ _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ int minRes = Integer.MAX_VALUE;
+ for (int j = 0; j < intValuesMV[i].length; j++) {
+ minRes = Math.min(minRes, intValuesMV[i][j]);
+ }
+ _intValuesSV[i] = minRes;
+ }
+ return _intValuesSV;
+ }
+
+ @Override
+ public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.LONG) {
+ return super.transformToLongValuesSV(projectionBlock);
+ }
+ if (_longValuesSV == null) {
+ _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ long minRes = Long.MAX_VALUE;
+ for (int j = 0; j < longValuesMV[i].length; j++) {
+ minRes = Math.min(minRes, longValuesMV[i][j]);
+ }
+ _longValuesSV[i] = minRes;
+ }
+ return _longValuesSV;
+ }
+
+ @Override
+ public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.FLOAT) {
+ return super.transformToFloatValuesSV(projectionBlock);
+ }
+ if (_floatValuesSV == null) {
+ _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ float minRes = Float.POSITIVE_INFINITY;
+ for (int j = 0; j < floatValuesMV[i].length; j++) {
+ minRes = Math.min(minRes, floatValuesMV[i][j]);
+ }
+ _floatValuesSV[i] = minRes;
+ }
+ return _floatValuesSV;
+ }
+
+ @Override
+ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.DOUBLE) {
+ return super.transformToDoubleValuesSV(projectionBlock);
+ }
+ if (_doubleValuesSV == null) {
+ _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ double minRes = Double.POSITIVE_INFINITY;
+ for (int j = 0; j < doubleValuesMV[i].length; j++) {
+ minRes = Math.min(minRes, doubleValuesMV[i][j]);
+ }
+ _doubleValuesSV[i] = minRes;
+ }
+ return _doubleValuesSV;
+ }
+
+ @Override
+ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
+ if (_argument.getResultMetadata().getDataType() != FieldSpec.DataType.STRING) {
+ return super.transformToStringValuesSV(projectionBlock);
+ }
+ if (_stringValuesSV == null) {
+ _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ String[][] stringValuesMV = _argument.transformToStringValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ String minRes = null;
+ for (int j = 0; j < stringValuesMV[i].length; j++) {
+ if (StringUtils.compare(minRes, stringValuesMV[i][j]) > 0) {
+ minRes = stringValuesMV[i][j];
+ }
+ }
+ _stringValuesSV[i] = minRes;
+ }
+ return _stringValuesSV;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java
new file mode 100644
index 0000000..ae5f6e2
--- /dev/null
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.core.common.DataSource;
+import org.apache.pinot.core.operator.blocks.ProjectionBlock;
+import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.plan.DocIdSetPlanNode;
+import org.apache.pinot.core.util.ArrayCopyUtils;
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+/**
+ * The ArraySumTransformFunction class implements array_sum function for multi-valued columns
+ *
+ * Sample queries:
+ * SELECT COUNT(*) FROM table WHERE array_sum(mvColumn) > 2
+ * SELECT COUNT(*) FROM table GROUP BY array_sum(mvColumn)
+ * SELECT SUM(array_sum(mvColumn)) FROM table
+ */
+public class ArraySumTransformFunction extends BaseTransformFunction {
+ public static final String FUNCTION_NAME = "array_sum";
+
+ private long[] _longResults;
+ private double[] _doubleResults;
+ private TransformFunction _argument;
+ private TransformResultMetadata _resultMetadata;
+
+ @Override
+ public String getName() {
+ return FUNCTION_NAME;
+ }
+
+ @Override
+ public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
+ // Check that there is only 1 argument
+ if (arguments.size() != 1) {
+ throw new IllegalArgumentException("Exactly 1 argument is required for ARRAY_AVERAGE transform function");
+ }
+
+ // Check that the argument is a multi-valued column or transform function
+ TransformFunction firstArgument = arguments.get(0);
+ if (firstArgument instanceof LiteralTransformFunction || firstArgument.getResultMetadata().isSingleValue()) {
+ throw new IllegalArgumentException(
+ "The argument of ARRAY_AVERAGE transform function must be a multi-valued column or a transform function");
+ }
+ FieldSpec.DataType resultDataType;
+ switch (firstArgument.getResultMetadata().getDataType()) {
+ case INT:
+ case LONG:
+ resultDataType = FieldSpec.DataType.LONG;
+ break;
+ case FLOAT:
+ case DOUBLE:
+ resultDataType = FieldSpec.DataType.DOUBLE;
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "The argument of ARRAY_AVERAGE transform function must be numeric");
+ }
+ _resultMetadata = new TransformResultMetadata(resultDataType, true, false);
+ _argument = firstArgument;
+ }
+
+ @Override
+ public TransformResultMetadata getResultMetadata() {
+ return _resultMetadata;
+ }
+
+ @Override
+ public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
+ if (_longResults == null) {
+ _longResults = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+ int length = projectionBlock.getNumDocs();
+ long sumRes;
+ switch (_argument.getResultMetadata().getDataType()) {
+ case INT:
+ case LONG:
+ long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ sumRes = 0;
+ for (int j = 0; j < longValuesMV[i].length; j++) {
+ sumRes += longValuesMV[i][j];
+ }
+ _longResults[i] = sumRes;
+ }
+ break;
+ case FLOAT:
+ case DOUBLE:
+ double[] doubleValues = transformToDoubleValuesSV(projectionBlock);
+ ArrayCopyUtils.copy(doubleValues, _longResults, length);
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ return _longResults;
+ }
+
+ @Override
+ public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
+ if (_doubleResults == null) {
+ _doubleResults = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ }
+
+ int length = projectionBlock.getNumDocs();
+ double sumRes;
+ switch (_argument.getResultMetadata().getDataType()) {
+ case INT:
+ case LONG:
+ long[] longValues = transformToLongValuesSV(projectionBlock);
+ ArrayCopyUtils.copy(longValues, _doubleResults, length);
+ break;
+ case FLOAT:
+ case DOUBLE:
+ double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ sumRes = 0;
+ for (int j = 0; j < doubleValuesMV[i].length; j++) {
+ sumRes += doubleValuesMV[i][j];
+ }
+ _doubleResults[i] = sumRes;
+ }
+ break;
+ default:
+ throw new IllegalStateException();
+ }
+ return _doubleResults;
+ }
+}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
index e70e5ed..4917708 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TransformFunctionFactory.java
@@ -92,6 +92,12 @@ public class TransformFunctionFactory {
put(TransformFunctionType.MAPVALUE.getName().toLowerCase(), MapValueTransformFunction.class);
put(TransformFunctionType.INIDSET.getName().toLowerCase(), InIdSetTransformFunction.class);
+ // Array functions
+ put(TransformFunctionType.ARRAY_AVERAGE.getName().toLowerCase(), ArrayAverageTransformFunction.class);
+ put(TransformFunctionType.ARRAY_MAX.getName().toLowerCase(), ArrayMaxTransformFunction.class);
+ put(TransformFunctionType.ARRAY_MIN.getName().toLowerCase(), ArrayMinTransformFunction.class);
+ put(TransformFunctionType.ARRAY_SUM.getName().toLowerCase(), ArraySumTransformFunction.class);
+
put(TransformFunctionType.GROOVY.getName().toLowerCase(), GroovyTransformFunction.class);
put(TransformFunctionType.CASE.getName().toLowerCase(), CaseTransformFunction.class);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunctionTest.java
new file mode 100644
index 0000000..0971adb
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunctionTest.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public class ArrayAverageTransformFunctionTest extends ArrayBaseTransformFunctionTest {
+
+ @Override
+ String getFunctionName() {
+ return ArrayAverageTransformFunction.FUNCTION_NAME;
+ }
+
+ @Override
+ Object getExpectResult(int[] intArrary) {
+ double sumRes = 0;
+ for (int v : intArrary) {
+ sumRes += v;
+ }
+ return sumRes / intArrary.length;
+ }
+
+ @Override
+ Class getArrayFunctionClass() {
+ return ArrayAverageTransformFunction.class;
+ }
+
+ @Override
+ FieldSpec.DataType getResultDataType(FieldSpec.DataType inputDataType) {
+ return FieldSpec.DataType.DOUBLE;
+ }
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayBaseTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayBaseTransformFunctionTest.java
new file mode 100644
index 0000000..3f5f02c
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayBaseTransformFunctionTest.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import org.apache.pinot.core.query.exception.BadQueryRequestException;
+import org.apache.pinot.core.query.request.context.ExpressionContext;
+import org.apache.pinot.core.query.request.context.utils.QueryContextConverterUtils;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+public abstract class ArrayBaseTransformFunctionTest extends BaseTransformFunctionTest {
+
+ @Test
+ public void testArrayTransformFunction() {
+ ExpressionContext expression =
+ QueryContextConverterUtils.getExpression(String.format("%s(%s)", getFunctionName(), INT_MV_COLUMN));
+ TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
+ Assert.assertEquals(transformFunction.getClass().getName(), getArrayFunctionClass().getName());
+ Assert.assertEquals(transformFunction.getName(), getFunctionName());
+ Assert.assertEquals(transformFunction.getResultMetadata().getDataType(), getResultDataType(FieldSpec.DataType.INT));
+ Assert.assertTrue(transformFunction.getResultMetadata().isSingleValue());
+ Assert.assertFalse(transformFunction.getResultMetadata().hasDictionary());
+
+ switch (getResultDataType(FieldSpec.DataType.INT)) {
+ case INT:
+ int[] intResults = transformFunction.transformToIntValuesSV(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Assert.assertEquals(intResults[i], getExpectResult(_intMVValues[i]));
+ }
+ break;
+ case LONG:
+ long[] longResults = transformFunction.transformToLongValuesSV(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Assert.assertEquals(longResults[i], getExpectResult(_intMVValues[i]));
+ }
+ break;
+ case FLOAT:
+ float[] floatResults = transformFunction.transformToFloatValuesSV(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Assert.assertEquals(floatResults[i], getExpectResult(_intMVValues[i]));
+ }
+ break;
+ case DOUBLE:
+ double[] doubleResults = transformFunction.transformToDoubleValuesSV(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Assert.assertEquals(doubleResults[i], getExpectResult(_intMVValues[i]));
+ }
+ break;
+ case STRING:
+ String[] stringResults = transformFunction.transformToStringValuesSV(_projectionBlock);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Assert.assertEquals(stringResults[i], getExpectResult(_intMVValues[i]));
+ }
+ break;
+ }
+ }
+
+ @Test(dataProvider = "testIllegalArguments", expectedExceptions = {BadQueryRequestException.class})
+ public void testIllegalArguments(String expressionStr) {
+ ExpressionContext expression = QueryContextConverterUtils.getExpression(expressionStr);
+ TransformFunctionFactory.get(expression, _dataSourceMap);
+ }
+
+ @DataProvider(name = "testIllegalArguments")
+ public Object[][] testIllegalArguments() {
+ return new Object[][]{new Object[]{String.format("%s(%s,1)", getFunctionName(),
+ INT_MV_COLUMN)}, new Object[]{String.format("%s(2)", getFunctionName())}, new Object[]{String.format("%s(%s)",
+ getFunctionName(), LONG_SV_COLUMN)}};
+ }
+
+ abstract String getFunctionName();
+
+ abstract Object getExpectResult(int[] intArray);
+
+ abstract Class getArrayFunctionClass();
+
+ abstract FieldSpec.DataType getResultDataType(FieldSpec.DataType inputDataType);
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunctionTest.java
index 81bcdcf..049e573 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunctionTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunctionTest.java
@@ -27,35 +27,25 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-public class ArrayLengthTransformFunctionTest extends BaseTransformFunctionTest {
+public class ArrayLengthTransformFunctionTest extends ArrayBaseTransformFunctionTest {
- @Test
- public void testLengthTransformFunction() {
- ExpressionContext expression =
- QueryContextConverterUtils.getExpression(String.format("arrayLength(%s)", INT_MV_COLUMN));
- TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
- Assert.assertTrue(transformFunction instanceof ArrayLengthTransformFunction);
- Assert.assertEquals(transformFunction.getName(), ArrayLengthTransformFunction.FUNCTION_NAME);
- Assert.assertEquals(transformFunction.getResultMetadata().getDataType(), FieldSpec.DataType.INT);
- Assert.assertTrue(transformFunction.getResultMetadata().isSingleValue());
- Assert.assertFalse(transformFunction.getResultMetadata().hasDictionary());
+ @Override
+ String getFunctionName() {
+ return ArrayLengthTransformFunction.FUNCTION_NAME;
+ }
- int[] results = transformFunction.transformToIntValuesSV(_projectionBlock);
- for (int i = 0; i < NUM_ROWS; i++) {
- Assert.assertEquals(results[i], _intMVValues[i].length);
- }
+ @Override
+ Object getExpectResult(int[] intArrary) {
+ return intArrary.length;
}
- @Test(dataProvider = "testIllegalArguments", expectedExceptions = {BadQueryRequestException.class})
- public void testIllegalArguments(String expressionStr) {
- ExpressionContext expression = QueryContextConverterUtils.getExpression(expressionStr);
- TransformFunctionFactory.get(expression, _dataSourceMap);
+ @Override
+ Class getArrayFunctionClass() {
+ return ArrayLengthTransformFunction.class;
}
- @DataProvider(name = "testIllegalArguments")
- public Object[][] testIllegalArguments() {
- return new Object[][]{new Object[]{String.format("arrayLength(%s,1)",
- INT_MV_COLUMN)}, new Object[]{"arrayLength(2)"}, new Object[]{String.format("arrayLength(%s)",
- LONG_SV_COLUMN)}};
+ @Override
+ FieldSpec.DataType getResultDataType(FieldSpec.DataType inputDataType) {
+ return FieldSpec.DataType.INT;
}
}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunctionTest.java
new file mode 100644
index 0000000..1c98462
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunctionTest.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public class ArrayMaxTransformFunctionTest extends ArrayBaseTransformFunctionTest {
+
+ @Override
+ String getFunctionName() {
+ return ArrayMaxTransformFunction.FUNCTION_NAME;
+ }
+
+ @Override
+ Object getExpectResult(int[] intArrary) {
+ int maxValue = Integer.MIN_VALUE;
+ for (int v : intArrary) {
+ maxValue = Math.max(maxValue, v);
+ }
+ return maxValue;
+ }
+
+ @Override
+ Class getArrayFunctionClass() {
+ return ArrayMaxTransformFunction.class;
+ }
+
+ @Override
+ FieldSpec.DataType getResultDataType(FieldSpec.DataType inputDataType) {
+ return inputDataType;
+ }
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunctionTest.java
new file mode 100644
index 0000000..48da84b
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunctionTest.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public class ArrayMinTransformFunctionTest extends ArrayBaseTransformFunctionTest {
+
+ @Override
+ String getFunctionName() {
+ return ArrayMinTransformFunction.FUNCTION_NAME;
+ }
+
+ @Override
+ Object getExpectResult(int[] intArrary) {
+ int minValue = Integer.MAX_VALUE;
+ for (int v : intArrary) {
+ minValue = Math.min(minValue, v);
+ }
+ return minValue;
+ }
+
+ @Override
+ Class getArrayFunctionClass() {
+ return ArrayMinTransformFunction.class;
+ }
+
+ @Override
+ FieldSpec.DataType getResultDataType(FieldSpec.DataType inputDataType) {
+ return inputDataType;
+ }
+}
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunctionTest.java b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunctionTest.java
new file mode 100644
index 0000000..15e455b
--- /dev/null
+++ b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunctionTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.operator.transform.function;
+
+import org.apache.pinot.spi.data.FieldSpec;
+
+
+public class ArraySumTransformFunctionTest extends ArrayBaseTransformFunctionTest {
+
+ @Override
+ String getFunctionName() {
+ return ArraySumTransformFunction.FUNCTION_NAME;
+ }
+
+ @Override
+ Object getExpectResult(int[] intArrary) {
+ long sumRes = 0;
+ for (int v : intArrary) {
+ sumRes += v;
+ }
+ return sumRes;
+ }
+
+ @Override
+ Class getArrayFunctionClass() {
+ return ArraySumTransformFunction.class;
+ }
+
+ @Override
+ FieldSpec.DataType getResultDataType(FieldSpec.DataType inputDataType) {
+ switch (inputDataType) {
+ case INT:
+ case LONG:
+ return FieldSpec.DataType.LONG;
+ case FLOAT:
+ case DOUBLE:
+ return FieldSpec.DataType.DOUBLE;
+ }
+ throw new IllegalArgumentException("Unsupported input data type - " + inputDataType);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org