You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2022/03/30 14:10:57 UTC
[pinot] branch master updated: Avoid creation of max length result arrays (#8419)
This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 46b821c Avoid creation of max length result arrays (#8419)
46b821c is described below
commit 46b821c3127093c9db64da246a1ad42017f415ff
Author: Kartik Khare <kh...@gmail.com>
AuthorDate: Wed Mar 30 19:40:27 2022 +0530
Avoid creation of max length result arrays (#8419)
* Only create results array for length of input and not for maxLength
* Reuse result arrays if input length is less than existing array length
* Remove unused imports
* Don't create large arrays in ValueIn
* Refactor numDocs to length
* Fix missing result array
---
.../function/AdditionTransformFunction.java | 8 +-
.../function/ArrayAverageTransformFunction.java | 16 ++--
.../function/ArrayLengthTransformFunction.java | 18 ++---
.../function/ArrayMaxTransformFunction.java | 41 +++++++----
.../function/ArrayMinTransformFunction.java | 45 +++++++----
.../function/ArraySumTransformFunction.java | 9 ++-
.../transform/function/BaseTransformFunction.java | 56 ++++++++------
.../transform/function/CastTransformFunction.java | 86 +++++++++++-----------
.../DateTimeConversionTransformFunction.java | 15 ++--
.../function/DateTruncTransformFunction.java | 6 +-
.../function/DivisionTransformFunction.java | 9 +--
.../function/InIdSetTransformFunction.java | 8 +-
.../transform/function/InTransformFunction.java | 6 +-
.../function/JsonExtractKeyTransformFunction.java | 10 +--
.../function/LogicalOperatorTransformFunction.java | 8 +-
.../function/MapValueTransformFunction.java | 8 +-
.../function/ModuloTransformFunction.java | 9 +--
.../function/MultiplicationTransformFunction.java | 8 +-
.../function/ScalarTransformFunctionWrapper.java | 76 ++++++++++---------
.../function/SingleParamMathTransformFunction.java | 27 ++++---
.../function/SubtractionTransformFunction.java | 9 +--
.../function/TimeConversionTransformFunction.java | 9 ++-
.../function/TrigonometricTransformFunctions.java | 24 +++---
.../function/ValueInTransformFunction.java | 38 ++++++----
24 files changed, 301 insertions(+), 248 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java
index bfc4904..29e4a68 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -66,11 +65,12 @@ public class AdditionTransformFunction extends BaseTransformFunction {
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_sums == null) {
- _sums = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_sums == null || _sums.length < length) {
+ _sums = new double[length];
}
- int length = projectionBlock.getNumDocs();
Arrays.fill(_sums, 0, length, _literalSum);
for (TransformFunction transformFunction : _transformFunctions) {
double[] values = transformFunction.transformToDoubleValuesSV(projectionBlock);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java
index 0ccd058..a24a28f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java
@@ -22,7 +22,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -71,15 +70,16 @@ public class ArrayAverageTransformFunction extends BaseTransformFunction {
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_results == null) {
- _results = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_results == null || _results.length < length) {
+ _results = new double[length];
}
- int numDocs = projectionBlock.getNumDocs();
switch (_argument.getResultMetadata().getDataType().getStoredType()) {
case INT:
int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
double sumRes = 0;
for (int value : intValuesMV[i]) {
sumRes += value;
@@ -89,7 +89,7 @@ public class ArrayAverageTransformFunction extends BaseTransformFunction {
break;
case LONG:
long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
double sumRes = 0;
for (long value : longValuesMV[i]) {
sumRes += value;
@@ -99,7 +99,7 @@ public class ArrayAverageTransformFunction extends BaseTransformFunction {
break;
case FLOAT:
float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
double sumRes = 0;
for (float value : floatValuesMV[i]) {
sumRes += value;
@@ -109,7 +109,7 @@ public class ArrayAverageTransformFunction extends BaseTransformFunction {
break;
case DOUBLE:
double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
double sumRes = 0;
for (double value : doubleValuesMV[i]) {
sumRes += value;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java
index 75efe80..005250f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java
@@ -22,7 +22,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -68,39 +67,40 @@ public class ArrayLengthTransformFunction extends BaseTransformFunction {
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
- if (_results == null) {
- _results = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_results == null || _results.length < length) {
+ _results = new int[length];
}
- int numDocs = projectionBlock.getNumDocs();
switch (_argument.getResultMetadata().getDataType().getStoredType()) {
case INT:
int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_results[i] = intValuesMV[i].length;
}
break;
case LONG:
long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_results[i] = longValuesMV[i].length;
}
break;
case FLOAT:
float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_results[i] = floatValuesMV[i].length;
}
break;
case DOUBLE:
double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_results[i] = doubleValuesMV[i].length;
}
break;
case STRING:
String[][] stringValuesMV = _argument.transformToStringValuesMV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_results[i] = stringValuesMV[i].length;
}
break;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java
index 81d7479..0e93a26 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -79,10 +78,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.INT) {
return super.transformToIntValuesSV(projectionBlock);
}
- if (_intValuesSV == null) {
- _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_intValuesSV == null || _intValuesSV.length < length) {
+ _intValuesSV = new int[length];
+ }
int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
int maxRes = Integer.MIN_VALUE;
@@ -99,10 +100,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.LONG) {
return super.transformToLongValuesSV(projectionBlock);
}
- if (_longValuesSV == null) {
- _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_longValuesSV == null || _longValuesSV.length < length) {
+ _longValuesSV = new long[length];
+ }
long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
long maxRes = Long.MIN_VALUE;
@@ -119,10 +122,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.FLOAT) {
return super.transformToFloatValuesSV(projectionBlock);
}
- if (_floatValuesSV == null) {
- _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_floatValuesSV == null || _floatValuesSV.length < length) {
+ _floatValuesSV = new float[length];
+ }
float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
float maxRes = Float.NEGATIVE_INFINITY;
@@ -139,10 +144,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.DOUBLE) {
return super.transformToDoubleValuesSV(projectionBlock);
}
- if (_doubleValuesSV == null) {
- _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_doubleValuesSV == null || _doubleValuesSV.length < length) {
+ _doubleValuesSV = new double[length];
+ }
double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
double maxRes = Double.NEGATIVE_INFINITY;
@@ -159,10 +166,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.STRING) {
return super.transformToStringValuesSV(projectionBlock);
}
- if (_stringValuesSV == null) {
- _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_stringValuesSV == null || _stringValuesSV.length < length) {
+ _stringValuesSV = new String[length];
+ }
String[][] stringValuesMV = _argument.transformToStringValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
String maxRes = null;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java
index d3de1da..90b529a 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -79,10 +78,12 @@ public class ArrayMinTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.INT) {
return super.transformToIntValuesSV(projectionBlock);
}
- if (_intValuesSV == null) {
- _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_intValuesSV == null || _intValuesSV.length < length) {
+ _intValuesSV = new int[length];
+ }
int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
int minRes = Integer.MAX_VALUE;
@@ -99,10 +100,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.LONG) {
return super.transformToLongValuesSV(projectionBlock);
}
- if (_longValuesSV == null) {
- _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_longValuesSV == null || _longValuesSV.length < length) {
+ _longValuesSV = new long[length];
+ }
+
long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
long minRes = Long.MAX_VALUE;
@@ -119,10 +123,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.FLOAT) {
return super.transformToFloatValuesSV(projectionBlock);
}
- if (_floatValuesSV == null) {
- _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_floatValuesSV == null || _floatValuesSV.length < length) {
+ _floatValuesSV = new float[length];
+ }
+
float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
float minRes = Float.POSITIVE_INFINITY;
@@ -139,10 +146,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.DOUBLE) {
return super.transformToDoubleValuesSV(projectionBlock);
}
- if (_doubleValuesSV == null) {
- _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_doubleValuesSV == null || _doubleValuesSV.length < length) {
+ _doubleValuesSV = new double[length];
+ }
+
double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
double minRes = Double.POSITIVE_INFINITY;
@@ -159,10 +169,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction {
if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.STRING) {
return super.transformToStringValuesSV(projectionBlock);
}
- if (_stringValuesSV == null) {
- _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
+
int length = projectionBlock.getNumDocs();
+
+ if (_stringValuesSV == null || _stringValuesSV.length < length) {
+ _stringValuesSV = new String[length];
+ }
+
String[][] stringValuesMV = _argument.transformToStringValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
String minRes = null;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java
index 33ae2a1..8faf4a0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java
@@ -22,7 +22,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -71,10 +70,12 @@ public class ArraySumTransformFunction extends BaseTransformFunction {
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_results == null) {
- _results = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
int length = projectionBlock.getNumDocs();
+
+ if (_results == null || _results.length < length) {
+ _results = new double[length];
+ }
+
double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
double sumRes = 0;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
index b7a3d91..45d7f0e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
@@ -21,7 +21,6 @@ package org.apache.pinot.core.operator.transform.function;
import com.google.common.base.Preconditions;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.ArrayCopyUtils;
@@ -81,11 +80,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_intValuesSV == null) {
- _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ _intValuesSV = new int[length];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -117,11 +117,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_longValuesSV == null) {
- _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ _longValuesSV = new long[length];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -153,11 +154,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_floatValuesSV == null) {
- _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ _floatValuesSV = new float[length];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -189,11 +191,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_doubleValuesSV == null) {
- _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ _doubleValuesSV = new double[length];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -225,11 +228,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_stringValuesSV == null) {
- _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ _stringValuesSV = new String[length];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -265,11 +269,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_byteValuesSV == null) {
- _byteValuesSV = new byte[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ _byteValuesSV = new byte[length][];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -284,11 +289,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_intValuesMV == null) {
- _intValuesMV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ _intValuesMV = new int[length][];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -350,11 +356,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_longValuesMV == null) {
- _longValuesMV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ _longValuesMV = new long[length][];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -416,11 +423,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_floatValuesMV == null) {
- _floatValuesMV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ _floatValuesMV = new float[length][];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -482,11 +490,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_doubleValuesMV == null) {
- _doubleValuesMV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ _doubleValuesMV = new double[length][];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -548,11 +557,12 @@ public abstract class BaseTransformFunction implements TransformFunction {
@Override
public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_stringValuesMV == null) {
- _stringValuesMV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ _stringValuesMV = new String[length][];
}
- int length = projectionBlock.getNumDocs();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
index 503474e..35499b1 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.ArrayCopyUtils;
@@ -99,26 +98,26 @@ public class CastTransformFunction extends BaseTransformFunction {
if (resultStoredType == DataType.INT) {
return _transformFunction.transformToIntValuesSV(projectionBlock);
} else {
- if (_intValuesSV == null) {
- _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+ if (_intValuesSV == null || _intValuesSV.length < length) {
+ _intValuesSV = new int[length];
}
- int numDocs = projectionBlock.getNumDocs();
switch (resultStoredType) {
case LONG:
long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _intValuesSV, numDocs);
+ ArrayCopyUtils.copy(longValues, _intValuesSV, length);
break;
case FLOAT:
float[] floatValues = _transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _intValuesSV, numDocs);
+ ArrayCopyUtils.copy(floatValues, _intValuesSV, length);
break;
case DOUBLE:
double[] doubleValues = _transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _intValuesSV, numDocs);
+ ArrayCopyUtils.copy(doubleValues, _intValuesSV, length);
break;
case STRING:
String[] stringValues = _transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _intValuesSV, numDocs);
+ ArrayCopyUtils.copy(stringValues, _intValuesSV, length);
break;
default:
throw new IllegalStateException();
@@ -134,26 +133,27 @@ public class CastTransformFunction extends BaseTransformFunction {
if (resultStoredType == DataType.LONG) {
return _transformFunction.transformToLongValuesSV(projectionBlock);
} else {
- if (_longValuesSV == null) {
- _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_longValuesSV == null || _longValuesSV.length < length) {
+ _longValuesSV = new long[length];
}
- int numDocs = projectionBlock.getNumDocs();
switch (resultStoredType) {
case INT:
int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _longValuesSV, numDocs);
+ ArrayCopyUtils.copy(intValues, _longValuesSV, length);
break;
case FLOAT:
float[] floatValues = _transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _longValuesSV, numDocs);
+ ArrayCopyUtils.copy(floatValues, _longValuesSV, length);
break;
case DOUBLE:
double[] doubleValues = _transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _longValuesSV, numDocs);
+ ArrayCopyUtils.copy(doubleValues, _longValuesSV, length);
break;
case STRING:
String[] stringValues = _transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _longValuesSV, numDocs);
+ ArrayCopyUtils.copy(stringValues, _longValuesSV, length);
break;
default:
throw new IllegalStateException();
@@ -169,26 +169,27 @@ public class CastTransformFunction extends BaseTransformFunction {
if (resultStoredType == DataType.FLOAT) {
return _transformFunction.transformToFloatValuesSV(projectionBlock);
} else {
- if (_floatValuesSV == null) {
- _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_floatValuesSV == null || _floatValuesSV.length < length) {
+ _floatValuesSV = new float[length];
}
- int numDocs = projectionBlock.getNumDocs();
switch (resultStoredType) {
case INT:
int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _floatValuesSV, numDocs);
+ ArrayCopyUtils.copy(intValues, _floatValuesSV, length);
break;
case LONG:
long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _floatValuesSV, numDocs);
+ ArrayCopyUtils.copy(longValues, _floatValuesSV, length);
break;
case DOUBLE:
double[] doubleValues = _transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _floatValuesSV, numDocs);
+ ArrayCopyUtils.copy(doubleValues, _floatValuesSV, length);
break;
case STRING:
String[] stringValues = _transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _floatValuesSV, numDocs);
+ ArrayCopyUtils.copy(stringValues, _floatValuesSV, length);
break;
default:
throw new IllegalStateException();
@@ -204,26 +205,27 @@ public class CastTransformFunction extends BaseTransformFunction {
if (resultStoredType == DataType.DOUBLE) {
return _transformFunction.transformToDoubleValuesSV(projectionBlock);
} else {
- if (_doubleValuesSV == null) {
- _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_doubleValuesSV == null || _doubleValuesSV.length < length) {
+ _doubleValuesSV = new double[length];
}
- int numDocs = projectionBlock.getNumDocs();
switch (resultStoredType) {
case INT:
int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _doubleValuesSV, numDocs);
+ ArrayCopyUtils.copy(intValues, _doubleValuesSV, length);
break;
case LONG:
long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _doubleValuesSV, numDocs);
+ ArrayCopyUtils.copy(longValues, _doubleValuesSV, length);
break;
case FLOAT:
float[] floatValues = _transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _doubleValuesSV, numDocs);
+ ArrayCopyUtils.copy(floatValues, _doubleValuesSV, length);
break;
case STRING:
String[] stringValues = _transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _doubleValuesSV, numDocs);
+ ArrayCopyUtils.copy(stringValues, _doubleValuesSV, length);
break;
default:
throw new IllegalStateException();
@@ -237,23 +239,23 @@ public class CastTransformFunction extends BaseTransformFunction {
// When casting to types other than STRING, need to first read as the result type then convert to string values
DataType resultDataType = _resultMetadata.getDataType();
DataType resultStoredType = resultDataType.getStoredType();
- int numDocs = projectionBlock.getNumDocs();
+ int length = projectionBlock.getNumDocs();
if (resultStoredType == DataType.STRING) {
// Specialize BOOlEAN and TIMESTAMP when casting to STRING
DataType inputDataType = _transformFunction.getResultMetadata().getDataType();
if (inputDataType.getStoredType() != inputDataType) {
- if (_stringValuesSV == null) {
- _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ if (_stringValuesSV == null || _stringValuesSV.length < length) {
+ _stringValuesSV = new String[length];
}
if (inputDataType == DataType.BOOLEAN) {
int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_stringValuesSV[i] = Boolean.toString(intValues[i] == 1);
}
} else {
assert inputDataType == DataType.TIMESTAMP;
long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_stringValuesSV[i] = new Timestamp(longValues[i]).toString();
}
}
@@ -262,35 +264,35 @@ public class CastTransformFunction extends BaseTransformFunction {
return _transformFunction.transformToStringValuesSV(projectionBlock);
}
} else {
- if (_stringValuesSV == null) {
- _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ if (_stringValuesSV == null || _stringValuesSV.length < length) {
+ _stringValuesSV = new String[length];
}
switch (resultDataType) {
case INT:
int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _stringValuesSV, numDocs);
+ ArrayCopyUtils.copy(intValues, _stringValuesSV, length);
break;
case LONG:
long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _stringValuesSV, numDocs);
+ ArrayCopyUtils.copy(longValues, _stringValuesSV, length);
break;
case FLOAT:
float[] floatValues = _transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _stringValuesSV, numDocs);
+ ArrayCopyUtils.copy(floatValues, _stringValuesSV, length);
break;
case DOUBLE:
double[] doubleValues = _transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _stringValuesSV, numDocs);
+ ArrayCopyUtils.copy(doubleValues, _stringValuesSV, length);
break;
case BOOLEAN:
intValues = _transformFunction.transformToIntValuesSV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_stringValuesSV[i] = Boolean.toString(intValues[i] == 1);
}
break;
case TIMESTAMP:
longValues = _transformFunction.transformToLongValuesSV(projectionBlock);
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
_stringValuesSV[i] = new Timestamp(longValues[i]).toString();
}
break;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java
index 99500bd..0c7c1cd 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java
@@ -28,7 +28,6 @@ import org.apache.pinot.core.operator.transform.transformer.datetime.EpochToEpoc
import org.apache.pinot.core.operator.transform.transformer.datetime.EpochToSDFTransformer;
import org.apache.pinot.core.operator.transform.transformer.datetime.SDFToEpochTransformer;
import org.apache.pinot.core.operator.transform.transformer.datetime.SDFToSDFTransformer;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.DateTimeFieldSpec;
@@ -127,11 +126,12 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction {
@Override
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
if (_resultMetadata == LONG_SV_NO_DICTIONARY_METADATA) {
- if (_longOutputTimes == null) {
- _longOutputTimes = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_longOutputTimes == null || _longOutputTimes.length < length) {
+ _longOutputTimes = new long[length];
}
- int length = projectionBlock.getNumDocs();
if (_dateTimeTransformer instanceof EpochToEpochTransformer) {
EpochToEpochTransformer dateTimeTransformer = (EpochToEpochTransformer) _dateTimeTransformer;
dateTimeTransformer
@@ -150,11 +150,12 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction {
@Override
public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
if (_resultMetadata == STRING_SV_NO_DICTIONARY_METADATA) {
- if (_stringOutputTimes == null) {
- _stringOutputTimes = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_stringOutputTimes == null || _stringOutputTimes.length < length) {
+ _stringOutputTimes = new String[length];
}
- int length = projectionBlock.getNumDocs();
if (_dateTimeTransformer instanceof EpochToSDFTransformer) {
EpochToSDFTransformer dateTimeTransformer = (EpochToSDFTransformer) _dateTimeTransformer;
dateTimeTransformer
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java
index 70e5e0d..8b4b29b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java
@@ -26,7 +26,6 @@ import org.apache.pinot.common.function.DateTimeUtils;
import org.apache.pinot.common.function.TimeZoneKey;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.joda.time.DateTimeField;
@@ -130,11 +129,12 @@ public class DateTruncTransformFunction extends BaseTransformFunction {
@Override
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_longOutputTimes == null) {
- _longOutputTimes = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ _longOutputTimes = new long[length];
}
- int length = projectionBlock.getNumDocs();
long[] input = _mainTransformFunction.transformToLongValuesSV(projectionBlock);
for (int i = 0; i < length; i++) {
_longOutputTimes[i] = _outputTimeUnit
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java
index dad8345..f65e5c2 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -77,12 +76,12 @@ public class DivisionTransformFunction extends BaseTransformFunction {
@SuppressWarnings("Duplicates")
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_quotients == null) {
- _quotients = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
-
int length = projectionBlock.getNumDocs();
+ if (_quotients == null || _quotients.length < length) {
+ _quotients = new double[length];
+ }
+
if (_firstTransformFunction == null) {
Arrays.fill(_quotients, 0, length, _firstLiteral);
} else {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java
index 4dcff5a..5b7f66e 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java
@@ -25,7 +25,6 @@ import java.util.Map;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.core.query.utils.idset.IdSet;
import org.apache.pinot.core.query.utils.idset.IdSets;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -75,11 +74,12 @@ public class InIdSetTransformFunction extends BaseTransformFunction {
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
- if (_results == null) {
- _results = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_results == null || _results.length < length) {
+ _results = new int[length];
}
- int length = projectionBlock.getNumDocs();
DataType storedType = _transformFunction.getResultMetadata().getDataType().getStoredType();
switch (storedType) {
case INT:
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
index 6245108..0172035 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java
@@ -31,7 +31,6 @@ import java.util.Set;
import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.ByteArray;
@@ -139,13 +138,14 @@ public class InTransformFunction extends BaseTransformFunction {
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_intValuesSV == null) {
- _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ _intValuesSV = new int[length];
} else {
Arrays.fill(_intValuesSV, 0);
}
- int length = projectionBlock.getNumDocs();
TransformResultMetadata mainFunctionMetadata = _mainFunction.getResultMetadata();
DataType storedType = mainFunctionMetadata.getDataType().getStoredType();
if (_valueSet != null) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java
index b5864c3..55a8d21 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java
@@ -29,7 +29,6 @@ import java.util.Map;
import org.apache.pinot.common.function.JsonPathCache;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -86,13 +85,14 @@ public class JsonExtractKeyTransformFunction extends BaseTransformFunction {
@Override
public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) {
- if (_stringValuesMV == null) {
- _stringValuesMV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ int length = projectionBlock.getNumDocs();
+
+ if (_stringValuesMV == null || _stringValuesSV.length < length) {
+ _stringValuesMV = new String[length][];
}
String[] jsonStrings = _jsonFieldTransformFunction.transformToStringValuesSV(projectionBlock);
- int numDocs = projectionBlock.getNumDocs();
- for (int i = 0; i < numDocs; i++) {
+ for (int i = 0; i < length; i++) {
List<String> values = JSON_PARSER_CONTEXT.parse(jsonStrings[i]).read(_jsonPath);
_stringValuesMV[i] = values.toArray(new String[0]);
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java
index 3fe232c..c96f4b9 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.utils.ArrayCopyUtils;
@@ -60,10 +59,11 @@ public abstract class LogicalOperatorTransformFunction extends BaseTransformFunc
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
- if (_results == null) {
- _results = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
int numDocs = projectionBlock.getNumDocs();
+
+ if (_results == null || _results.length < numDocs) {
+ _results = new int[numDocs];
+ }
ArrayCopyUtils.copy(_arguments.get(0).transformToIntValuesSV(projectionBlock), _results, numDocs);
int numArguments = _arguments.size();
for (int i = 1; i < numArguments; i++) {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java
index 395b20d..8ccac42 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
@@ -97,13 +96,14 @@ public class MapValueTransformFunction extends BaseTransformFunction {
@Override
public int[] transformToDictIdsSV(ProjectionBlock projectionBlock) {
- if (_dictIds == null) {
- _dictIds = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_dictIds == null || _dictIds.length < length) {
+ _dictIds = new int[length];
}
int[][] keyDictIdsMV = _keyColumnFunction.transformToDictIdsMV(projectionBlock);
int[][] valueDictIdsMV = _valueColumnFunction.transformToDictIdsMV(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
int[] keyDictIds = keyDictIdsMV[i];
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java
index 4339eec..c660e1b 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -77,12 +76,12 @@ public class ModuloTransformFunction extends BaseTransformFunction {
@SuppressWarnings("Duplicates")
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_modulos == null) {
- _modulos = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
-
int length = projectionBlock.getNumDocs();
+ if (_modulos == null || _modulos.length < length) {
+ _modulos = new double[length];
+ }
+
if (_firstTransformFunction == null) {
Arrays.fill(_modulos, 0, length, _firstLiteral);
} else {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java
index 63d3986..095e023 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -66,11 +65,12 @@ public class MultiplicationTransformFunction extends BaseTransformFunction {
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_products == null) {
- _products = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_products == null || _products.length < length) {
+ _products = new double[length];
}
- int length = projectionBlock.getNumDocs();
Arrays.fill(_products, 0, length, _literalProduct);
for (TransformFunction transformFunction : _transformFunctions) {
double[] values = transformFunction.transformToDoubleValuesSV(projectionBlock);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
index 34d44d3..39755a0 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java
@@ -29,7 +29,6 @@ import org.apache.pinot.common.function.FunctionUtils;
import org.apache.pinot.common.utils.PinotDataType;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -125,11 +124,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
return super.transformToIntValuesSV(projectionBlock);
}
- if (_intResults == null) {
- _intResults = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_intResults == null || _intResults.length < length) {
+ _intResults = new int[length];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -144,11 +144,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
return super.transformToLongValuesSV(projectionBlock);
}
- if (_longResults == null) {
- _longResults = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_longResults == null || _longResults.length < length) {
+ _longResults = new long[length];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -163,11 +164,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
return super.transformToFloatValuesSV(projectionBlock);
}
- if (_floatResults == null) {
- _floatResults = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_floatResults == null || _floatResults.length < length) {
+ _floatResults = new float[length];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -182,11 +184,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
return super.transformToDoubleValuesSV(projectionBlock);
}
- if (_doubleResults == null) {
- _doubleResults = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_doubleResults == null || _doubleResults.length < length) {
+ _doubleResults = new double[length];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -201,11 +204,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
return super.transformToStringValuesSV(projectionBlock);
}
- if (_stringResults == null) {
- _stringResults = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_stringResults == null || _stringResults.length < length) {
+ _stringResults = new String[length];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -222,11 +226,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.BYTES) {
return super.transformToBytesValuesSV(projectionBlock);
}
- if (_bytesResults == null) {
- _bytesResults = new byte[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ int length = projectionBlock.getNumDocs();
+
+ if (_bytesResults == null || _bytesResults.length < length) {
+ _bytesResults = new byte[length][];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -241,11 +246,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.INT) {
return super.transformToIntValuesMV(projectionBlock);
}
+ int length = projectionBlock.getNumDocs();
+
if (_intMVResults == null) {
- _intMVResults = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ _intMVResults = new int[length][];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -260,11 +266,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) {
return super.transformToLongValuesMV(projectionBlock);
}
- if (_longMVResults == null) {
- _longMVResults = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ int length = projectionBlock.getNumDocs();
+
+ if (_longMVResults == null || _longMVResults.length < length) {
+ _longMVResults = new long[length][];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -279,11 +286,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) {
return super.transformToFloatValuesMV(projectionBlock);
}
- if (_floatMVResults == null) {
- _floatMVResults = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ int length = projectionBlock.getNumDocs();
+
+ if (_floatMVResults == null || _floatMVResults.length < length) {
+ _floatMVResults = new float[length][];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -298,11 +306,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) {
return super.transformToDoubleValuesMV(projectionBlock);
}
- if (_doubleMVResults == null) {
- _doubleMVResults = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ int length = projectionBlock.getNumDocs();
+
+ if (_doubleMVResults == null || _doubleMVResults.length < length) {
+ _doubleMVResults = new double[length][];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
@@ -317,11 +326,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction {
if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) {
return super.transformToStringValuesMV(projectionBlock);
}
- if (_stringMVResults == null) {
- _stringMVResults = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ int length = projectionBlock.getNumDocs();
+
+ if (_stringMVResults == null || _stringMVResults.length < length) {
+ _stringMVResults = new String[length][];
}
getNonLiteralValues(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
for (int j = 0; j < _numNonLiteralArguments; j++) {
_arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i];
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java
index bcc733e..ff1dafb 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java
@@ -32,7 +32,6 @@ import org.apache.pinot.segment.spi.datasource.DataSource;
*/
public abstract class SingleParamMathTransformFunction extends BaseTransformFunction {
private TransformFunction _transformFunction;
- protected double[] _results;
@Override
public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) {
@@ -56,13 +55,13 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
- if (_results == null || _results.length < length) {
- _results = new double[length];
+ if (_doubleValuesSV == null || _doubleValuesSV.length < length) {
+ _doubleValuesSV = new double[length];
}
double[] values = _transformFunction.transformToDoubleValuesSV(projectionBlock);
- applyMathOperator(values, length);
- return _results;
+ applyMathOperator(values, projectionBlock.getNumDocs());
+ return _doubleValuesSV;
}
abstract protected void applyMathOperator(double[] values, int length);
@@ -78,7 +77,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.abs(values[i]);
+ _doubleValuesSV[i] = Math.abs(values[i]);
}
}
}
@@ -94,7 +93,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.ceil(values[i]);
+ _doubleValuesSV[i] = Math.ceil(values[i]);
}
}
}
@@ -110,7 +109,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.exp(values[i]);
+ _doubleValuesSV[i] = Math.exp(values[i]);
}
}
}
@@ -126,7 +125,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.floor(values[i]);
+ _doubleValuesSV[i] = Math.floor(values[i]);
}
}
}
@@ -142,7 +141,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.log(values[i]);
+ _doubleValuesSV[i] = Math.log(values[i]);
}
}
}
@@ -159,7 +158,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.log(values[i]) / LOG_BASE;
+ _doubleValuesSV[i] = Math.log(values[i]) / LOG_BASE;
}
}
}
@@ -175,7 +174,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.log10(values[i]);
+ _doubleValuesSV[i] = Math.log10(values[i]);
}
}
}
@@ -191,7 +190,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.sqrt(values[i]);
+ _doubleValuesSV[i] = Math.sqrt(values[i]);
}
}
}
@@ -207,7 +206,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.signum(values[i]);
+ _doubleValuesSV[i] = Math.signum(values[i]);
}
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java
index bd7b50a..c910269 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -77,12 +76,12 @@ public class SubtractionTransformFunction extends BaseTransformFunction {
@SuppressWarnings("Duplicates")
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_differences == null) {
- _differences = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL];
- }
-
int length = projectionBlock.getNumDocs();
+ if (_differences == null || _differences.length < length) {
+ _differences = new double[length];
+ }
+
if (_firstTransformFunction == null) {
Arrays.fill(_differences, 0, length, _firstLiteral);
} else {
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java
index ac1a535..2061b8f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java
@@ -25,7 +25,6 @@ import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.core.operator.transform.transformer.timeunit.TimeUnitTransformer;
import org.apache.pinot.core.operator.transform.transformer.timeunit.TimeUnitTransformerFactory;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
@@ -68,12 +67,14 @@ public class TimeConversionTransformFunction extends BaseTransformFunction {
@Override
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
- if (_outputTimes == null) {
- _outputTimes = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL];
+ int length = projectionBlock.getNumDocs();
+
+ if (_outputTimes == null || _outputTimes.length < length) {
+ _outputTimes = new long[length];
}
_timeUnitTransformer.transform(_mainTransformFunction.transformToLongValuesSV(projectionBlock), _outputTimes,
- projectionBlock.getNumDocs());
+ length);
return _outputTimes;
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TrigonometricTransformFunctions.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TrigonometricTransformFunctions.java
index e694df1..c7febb5 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TrigonometricTransformFunctions.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TrigonometricTransformFunctions.java
@@ -85,7 +85,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.toDegrees(values[i]);
+ _doubleValuesSV[i] = Math.toDegrees(values[i]);
}
}
}
@@ -101,7 +101,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.acos(values[i]);
+ _doubleValuesSV[i] = Math.acos(values[i]);
}
}
}
@@ -117,7 +117,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.tan(values[i]);
+ _doubleValuesSV[i] = Math.tan(values[i]);
}
}
}
@@ -133,7 +133,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.sinh(values[i]);
+ _doubleValuesSV[i] = Math.sinh(values[i]);
}
}
}
@@ -149,7 +149,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = 1.0 / Math.tan(values[i]);
+ _doubleValuesSV[i] = 1.0 / Math.tan(values[i]);
}
}
}
@@ -165,7 +165,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.atan(values[i]);
+ _doubleValuesSV[i] = Math.atan(values[i]);
}
}
}
@@ -181,7 +181,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.cos(values[i]);
+ _doubleValuesSV[i] = Math.cos(values[i]);
}
}
}
@@ -197,7 +197,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.asin(values[i]);
+ _doubleValuesSV[i] = Math.asin(values[i]);
}
}
}
@@ -213,7 +213,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.cosh(values[i]);
+ _doubleValuesSV[i] = Math.cosh(values[i]);
}
}
}
@@ -229,7 +229,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.sin(values[i]);
+ _doubleValuesSV[i] = Math.sin(values[i]);
}
}
}
@@ -245,7 +245,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.tanh(values[i]);
+ _doubleValuesSV[i] = Math.tanh(values[i]);
}
}
}
@@ -261,7 +261,7 @@ public class TrigonometricTransformFunctions {
@Override
protected void applyMathOperator(double[] values, int length) {
for (int i = 0; i < length; i++) {
- _results[i] = Math.toRadians(values[i]);
+ _doubleValuesSV[i] = Math.toRadians(values[i]);
}
}
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java
index 0d1f20b..0fcf6a8 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java
@@ -41,7 +41,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
-import org.apache.pinot.core.plan.DocIdSetPlanNode;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.data.FieldSpec;
@@ -109,6 +108,8 @@ public class ValueInTransformFunction extends BaseTransformFunction {
@Override
public int[][] transformToDictIdsMV(ProjectionBlock projectionBlock) {
+ int length = projectionBlock.getNumDocs();
+
if (_dictIdSet == null) {
_dictIdSet = new IntOpenHashSet();
assert _dictionary != null;
@@ -118,10 +119,11 @@ public class ValueInTransformFunction extends BaseTransformFunction {
_dictIdSet.add(dictId);
}
}
- _dictIds = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ if (_dictIds == null || _dictIds.length < length) {
+ _dictIds = new int[length][];
+ }
}
int[][] unFilteredDictIds = _mainTransformFunction.transformToDictIdsMV(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
_dictIds[i] = filterInts(_dictIdSet, unFilteredDictIds[i]);
}
@@ -134,15 +136,17 @@ public class ValueInTransformFunction extends BaseTransformFunction {
return super.transformToIntValuesMV(projectionBlock);
}
+ int length = projectionBlock.getNumDocs();
if (_intValueSet == null) {
_intValueSet = new IntOpenHashSet();
for (String inValue : _stringValueSet) {
_intValueSet.add(Integer.parseInt(inValue));
}
- _intValues = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ if (_intValues == null || _intValues.length < length) {
+ _intValues = new int[length][];
+ }
}
int[][] unFilteredIntValues = _mainTransformFunction.transformToIntValuesMV(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
_intValues[i] = filterInts(_intValueSet, unFilteredIntValues[i]);
}
@@ -155,15 +159,17 @@ public class ValueInTransformFunction extends BaseTransformFunction {
return super.transformToLongValuesMV(projectionBlock);
}
+ int length = projectionBlock.getNumDocs();
if (_longValueSet == null) {
_longValueSet = new LongOpenHashSet();
for (String inValue : _stringValueSet) {
_longValueSet.add(Long.parseLong(inValue));
}
- _longValues = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ if (_longValues == null || _longValues.length < length) {
+ _longValues = new long[length][];
+ }
}
long[][] unFilteredLongValues = _mainTransformFunction.transformToLongValuesMV(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
_longValues[i] = filterLongs(_longValueSet, unFilteredLongValues[i]);
}
@@ -176,15 +182,17 @@ public class ValueInTransformFunction extends BaseTransformFunction {
return super.transformToFloatValuesMV(projectionBlock);
}
+ int length = projectionBlock.getNumDocs();
if (_floatValueSet == null) {
_floatValueSet = new FloatOpenHashSet();
for (String inValue : _stringValueSet) {
_floatValueSet.add(Float.parseFloat(inValue));
}
- _floatValues = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ if (_floatValues == null || _floatValues.length < length) {
+ _floatValues = new float[length][];
+ }
}
float[][] unFilteredFloatValues = _mainTransformFunction.transformToFloatValuesMV(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
_floatValues[i] = filterFloats(_floatValueSet, unFilteredFloatValues[i]);
}
@@ -197,15 +205,17 @@ public class ValueInTransformFunction extends BaseTransformFunction {
return super.transformToDoubleValuesMV(projectionBlock);
}
+ int length = projectionBlock.getNumDocs();
if (_doubleValueSet == null) {
_doubleValueSet = new DoubleOpenHashSet();
for (String inValue : _stringValueSet) {
_doubleValueSet.add(Double.parseDouble(inValue));
}
- _doubleValues = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ if (_doubleValues == null || _doubleValues.length < length) {
+ _doubleValues = new double[length][];
+ }
}
double[][] unFilteredDoubleValues = _mainTransformFunction.transformToDoubleValuesMV(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
_doubleValues[i] = filterDoubles(_doubleValueSet, unFilteredDoubleValues[i]);
}
@@ -218,11 +228,11 @@ public class ValueInTransformFunction extends BaseTransformFunction {
return super.transformToStringValuesMV(projectionBlock);
}
- if (_stringValues == null) {
- _stringValues = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][];
+ int length = projectionBlock.getNumDocs();
+ if (_stringValues == null || _stringValues.length < length) {
+ _stringValues = new String[length][];
}
String[][] unFilteredStringValues = _mainTransformFunction.transformToStringValuesMV(projectionBlock);
- int length = projectionBlock.getNumDocs();
for (int i = 0; i < length; i++) {
_stringValues[i] = filterStrings(_stringValueSet, unFilteredStringValues[i]);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org