You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ra...@apache.org on 2015/07/02 07:39:44 UTC
phoenix git commit: PHOENIX-2021 - Implement ARRAY_CAT built in
function (Dumindu Buddhika)
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-1.0 773d2c695 -> afb21ba11
PHOENIX-2021 - Implement ARRAY_CAT built in function (Dumindu Buddhika)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/afb21ba1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/afb21ba1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/afb21ba1
Branch: refs/heads/4.x-HBase-1.0
Commit: afb21ba1138d40faaa9461778ad92f5b77744b71
Parents: 773d2c6
Author: ramkrishna <ra...@gmail.com>
Authored: Thu Jul 2 11:09:06 2015 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Thu Jul 2 11:09:06 2015 +0530
----------------------------------------------------------------------
.../phoenix/end2end/ArrayAppendFunctionIT.java | 17 --
.../phoenix/expression/ExpressionType.java | 4 +-
.../function/ArrayAppendFunction.java | 53 +-----
.../function/ArrayModifierFunction.java | 155 +++++++++++++++---
.../function/ArrayPrependFunction.java | 54 +-----
.../phoenix/schema/types/PArrayDataType.java | 163 +++++++++++++++++--
6 files changed, 298 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/afb21ba1/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java
index 1957b3a..cf45724 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayAppendFunctionIT.java
@@ -497,23 +497,6 @@ public class ArrayAppendFunctionIT extends BaseHBaseManagedTimeIT {
}
@Test
- public void testArrayAppendFunctionIntegerWithNull() throws Exception {
- Connection conn = DriverManager.getConnection(getUrl());
- initTables(conn);
-
- ResultSet rs;
- rs = conn.createStatement().executeQuery("SELECT ARRAY_APPEND(NULL,NULL) FROM regions WHERE region_name = 'SF Bay Area'");
- assertTrue(rs.next());
-
- Integer[] integers = new Integer[]{2345, 46345, 23234, 456};
-
- Array array = conn.createArrayOf("INTEGER", integers);
-
- assertEquals(null, rs.getArray(1));
- assertFalse(rs.next());
- }
-
- @Test
public void testArrayAppendFunctionVarcharWithNull() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
initTables(conn);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/afb21ba1/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
index 4f98cb8..51f4089 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java
@@ -23,6 +23,7 @@ import org.apache.phoenix.expression.function.AbsFunction;
import org.apache.phoenix.expression.function.ArrayAllComparisonExpression;
import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression;
import org.apache.phoenix.expression.function.ArrayAppendFunction;
+import org.apache.phoenix.expression.function.ArrayConcatFunction;
import org.apache.phoenix.expression.function.ArrayElemRefExpression;
import org.apache.phoenix.expression.function.ArrayIndexFunction;
import org.apache.phoenix.expression.function.ArrayLengthFunction;
@@ -245,7 +246,8 @@ public enum ExpressionType {
LnFunction(LnFunction.class),
LogFunction(LogFunction.class),
ExpFunction(ExpFunction.class),
- PowerFunction(PowerFunction.class)
+ PowerFunction(PowerFunction.class),
+ ArrayConcatFunction(ArrayConcatFunction.class)
;
ExpressionType(Class<? extends Expression> clazz) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/afb21ba1/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java
index bf6c29f..8c7fa9f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayAppendFunction.java
@@ -20,18 +20,13 @@ package org.apache.phoenix.expression.function;
import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.phoenix.exception.DataExceedsCapacityException;
import org.apache.phoenix.expression.Expression;
-import org.apache.phoenix.expression.LiteralExpression;
import org.apache.phoenix.parse.FunctionParseNode;
-import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TypeMismatchException;
import org.apache.phoenix.schema.types.*;
-import org.apache.phoenix.schema.tuple.Tuple;
@FunctionParseNode.BuiltInFunction(name = ArrayAppendFunction.NAME, args = {
- @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class,
- PVarbinaryArray.class}),
+ @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, PVarbinaryArray.class}),
@FunctionParseNode.Argument(allowedTypes = {PVarbinary.class}, defaultValue = "null")})
public class ArrayAppendFunction extends ArrayModifierFunction {
@@ -45,54 +40,12 @@ public class ArrayAppendFunction extends ArrayModifierFunction {
}
@Override
- public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
-
- if (!getArrayExpr().evaluate(tuple, ptr)) {
- return false;
- } else if (ptr.getLength() == 0) {
- return true;
- }
- int arrayLength = PArrayDataType.getArrayLength(ptr, getBaseType(), getArrayExpr().getMaxLength());
-
- int length = ptr.getLength();
- int offset = ptr.getOffset();
- byte[] arrayBytes = ptr.get();
-
- if (!getElementExpr().evaluate(tuple, ptr) || ptr.getLength() == 0) {
- ptr.set(arrayBytes, offset, length);
- return true;
- }
-
- checkSizeCompatibility(ptr);
- coerceBytes(ptr);
- return PArrayDataType.appendItemToArray(ptr, length, offset, arrayBytes, getBaseType(), arrayLength, getMaxLength(), getArrayExpr().getSortOrder());
- }
-
- @Override
- public PDataType getDataType() {
- return children.get(0).getDataType();
- }
-
- @Override
- public Integer getMaxLength() {
- return this.children.get(0).getMaxLength();
- }
-
- @Override
- public SortOrder getSortOrder() {
- return getChildren().get(0).getSortOrder();
+ protected boolean modifierFunction(ImmutableBytesWritable ptr, int len, int offset, byte[] arrayBytes, PDataType baseDataType, int arrayLength, Integer maxLength, Expression arrayExp) {
+ return PArrayDataType.appendItemToArray(ptr, len, offset, arrayBytes, baseDataType, arrayLength, getMaxLength(), arrayExp.getSortOrder());
}
@Override
public String getName() {
return NAME;
}
-
- public Expression getArrayExpr() {
- return getChildren().get(0);
- }
-
- public Expression getElementExpr() {
- return getChildren().get(1);
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/afb21ba1/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java
index afd10e5..3177c29 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayModifierFunction.java
@@ -24,7 +24,9 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.exception.DataExceedsCapacityException;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.LiteralExpression;
+import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TypeMismatchException;
+import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.*;
public abstract class ArrayModifierFunction extends ScalarFunction {
@@ -34,42 +36,153 @@ public abstract class ArrayModifierFunction extends ScalarFunction {
public ArrayModifierFunction(List<Expression> children) throws TypeMismatchException {
super(children);
+ Expression arrayExpr = null;
+ PDataType baseDataType = null;
+ Expression otherExpr = null;
+ PDataType otherExpressionType = null;
+ if (getLHSExpr().getDataType().isArrayType()) {
+ arrayExpr = getLHSExpr();
+ baseDataType = getLHSBaseType();
+ otherExpr = getRHSExpr();
+ otherExpressionType = getRHSBaseType();
+ } else {
+ arrayExpr = getRHSExpr();
+ baseDataType = getRHSBaseType();
+ otherExpr = getLHSExpr();
+ otherExpressionType = getLHSBaseType();
+ }
+ if (getDataType() != null && !(otherExpr instanceof LiteralExpression && otherExpr.isNullable()) && !otherExpressionType.isCoercibleTo(baseDataType)) {
+ throw TypeMismatchException.newException(baseDataType, otherExpressionType);
+ }
- if (getDataType() != null && !(getElementExpr() instanceof LiteralExpression && getElementExpr().isNullable()) && !getElementDataType().isCoercibleTo(getBaseType())) {
- throw TypeMismatchException.newException(getBaseType(), getElementDataType());
+ // If the base type of an element is fixed width, make sure the element
+ // being appended will fit
+ if (getDataType() != null && otherExpressionType.getByteSize() == null
+ && otherExpressionType != null && baseDataType.isFixedWidth()
+ && otherExpressionType.isFixedWidth() && arrayExpr.getMaxLength() != null
+ && otherExpr.getMaxLength() != null
+ && otherExpr.getMaxLength() > arrayExpr.getMaxLength()) {
+ throw new DataExceedsCapacityException("Values are not size compatible");
}
+ // If the base type has a scale, make sure the element being appended has a
+ // scale less than or equal to it
+ if (getDataType() != null && arrayExpr.getScale() != null && otherExpr.getScale() != null
+ && otherExpr.getScale() > arrayExpr.getScale()) {
+ throw new DataExceedsCapacityException(baseDataType, arrayExpr.getMaxLength(),
+ arrayExpr.getScale());
+ }
+ }
- // If the base type of an element is fixed width, make sure the element being appended will fit
- if (getDataType() != null && getElementExpr().getDataType().getByteSize() == null && getElementDataType() != null && getBaseType().isFixedWidth() && getElementDataType().isFixedWidth() && getArrayExpr().getMaxLength() != null &&
- getElementExpr().getMaxLength() != null && getElementExpr().getMaxLength() > getArrayExpr().getMaxLength()) {
- throw new DataExceedsCapacityException("");
+ @Override
+ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
+ Expression arrayExpr = null;
+ PDataType baseDataType = null;
+ Expression otherExpr = null;
+ PDataType otherExpressionType = null;
+ if (getLHSExpr().getDataType().isArrayType()) {
+ arrayExpr = getLHSExpr();
+ baseDataType = getLHSBaseType();
+ otherExpr = getRHSExpr();
+ otherExpressionType = getRHSBaseType();
+ } else {
+ arrayExpr = getRHSExpr();
+ baseDataType = getRHSBaseType();
+ otherExpr = getLHSExpr();
+ otherExpressionType = getLHSBaseType();
}
- // If the base type has a scale, make sure the element being appended has a scale less than or equal to it
- if (getDataType() != null && getArrayExpr().getScale() != null && getElementExpr().getScale() != null &&
- getElementExpr().getScale() > getArrayExpr().getScale()) {
- throw new DataExceedsCapacityException(getBaseType(), getArrayExpr().getMaxLength(), getArrayExpr().getScale());
+ if (!arrayExpr.evaluate(tuple, ptr)) {
+ return false;
+ } else if (ptr.getLength() == 0) {
+ return true;
}
+ int arrayLength = PArrayDataType.getArrayLength(ptr, baseDataType, arrayExpr.getMaxLength());
+
+ int length = ptr.getLength();
+ int offset = ptr.getOffset();
+ byte[] arrayBytes = ptr.get();
+
+ otherExpr.evaluate(tuple, ptr);
+
+ checkSizeCompatibility(ptr, arrayExpr, baseDataType, otherExpr, otherExpressionType);
+ coerceBytes(ptr, arrayExpr, baseDataType, otherExpr, otherExpressionType);
+ return modifierFunction(ptr, length, offset, arrayBytes, baseDataType, arrayLength, getMaxLength(),
+ arrayExpr);
}
- protected void checkSizeCompatibility(ImmutableBytesWritable ptr) {
- if (!getBaseType().isSizeCompatible(ptr, null, getElementDataType(), getElementExpr().getMaxLength(), getElementExpr().getScale(), getArrayExpr().getMaxLength(), getArrayExpr().getScale())) {
- throw new DataExceedsCapacityException("");
+ // Override this method for various function implementations
+ protected boolean modifierFunction(ImmutableBytesWritable ptr, int len, int offset,
+ byte[] arrayBytes, PDataType baseDataType, int arrayLength, Integer maxLength,
+ Expression arrayExp) {
+ return false;
+ }
+
+ protected void checkSizeCompatibility(ImmutableBytesWritable ptr, Expression arrayExpr,
+ PDataType baseDataType, Expression otherExpr, PDataType otherExpressionType) {
+ if (!baseDataType.isSizeCompatible(ptr, null, otherExpressionType,
+ otherExpr.getMaxLength(), otherExpr.getScale(), arrayExpr.getMaxLength(),
+ arrayExpr.getScale())) {
+ throw new DataExceedsCapacityException("Values are not size compatible");
}
}
- protected void coerceBytes(ImmutableBytesWritable ptr) {
- getBaseType().coerceBytes(ptr, null, getElementDataType(), getElementExpr().getMaxLength(), getElementExpr().getScale(), getElementExpr().getSortOrder(), getArrayExpr().getMaxLength(), getArrayExpr().getScale(), getArrayExpr().getSortOrder());
+
+ protected void coerceBytes(ImmutableBytesWritable ptr, Expression arrayExpr,
+ PDataType baseDataType, Expression otherExpr, PDataType otherExpressionType) {
+ baseDataType.coerceBytes(ptr, null, otherExpressionType, otherExpr.getMaxLength(),
+ otherExpr.getScale(), otherExpr.getSortOrder(), arrayExpr.getMaxLength(),
+ arrayExpr.getScale(), arrayExpr.getSortOrder());
}
- public abstract Expression getArrayExpr();
+ public Expression getRHSExpr() {
+ return this.children.get(1);
+ }
- public abstract Expression getElementExpr();
+ public Expression getLHSExpr() {
+ return this.children.get(0);
+ }
- public PDataType getBaseType() {
- return PDataType.arrayBaseType(getArrayExpr().getDataType());
+ public PDataType getLHSBaseType() {
+ if (getLHSExpr().getDataType().isArrayType()) {
+ return PDataType.arrayBaseType(getLHSExpr().getDataType());
+ } else {
+ return getLHSExpr().getDataType();
+ }
}
- public PDataType getElementDataType() {
- return getElementExpr().getDataType();
+ public PDataType getRHSBaseType() {
+ if (getRHSExpr().getDataType().isArrayType()) {
+ return PDataType.arrayBaseType(getRHSExpr().getDataType());
+ } else {
+ return getRHSExpr().getDataType();
+ }
+ }
+
+ @Override
+ public PDataType getDataType() {
+ if (getLHSExpr().getDataType().isArrayType()) {
+ return getLHSExpr().getDataType();
+ } else {
+ return getRHSExpr().getDataType();
+ }
+ }
+
+
+ @Override
+ public Integer getMaxLength() {
+ if (getLHSExpr().getDataType().isArrayType()) {
+ return getLHSExpr().getMaxLength();
+ } else {
+ return getRHSExpr().getMaxLength();
+ }
+ }
+
+ @Override
+ public SortOrder getSortOrder() {
+ if (getLHSExpr().getDataType().isArrayType()) {
+ return getLHSExpr().getSortOrder();
+ } else {
+ return getRHSExpr().getSortOrder();
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/afb21ba1/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java
index 3cea4df..c2311fb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayPrependFunction.java
@@ -23,16 +23,13 @@ import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.parse.FunctionParseNode;
-import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TypeMismatchException;
-import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.*;
@FunctionParseNode.BuiltInFunction(name = ArrayPrependFunction.NAME, args = {
@FunctionParseNode.Argument(allowedTypes = {PVarbinary.class}),
- @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class,
- PVarbinaryArray.class})})
-public class ArrayPrependFunction extends ArrayModifierFunction {
+ @FunctionParseNode.Argument(allowedTypes = {PBinaryArray.class, PVarbinaryArray.class})})
+public class ArrayPrependFunction extends ArrayModifierFunction {
public static final String NAME = "ARRAY_PREPEND";
@@ -44,53 +41,14 @@ public class ArrayPrependFunction extends ArrayModifierFunction {
}
@Override
- public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
-
- if (!getArrayExpr().evaluate(tuple, ptr)) {
- return false;
- } else if (ptr.getLength() == 0) {
- return true;
- }
- int arrayLength = PArrayDataType.getArrayLength(ptr, getBaseType(), getArrayExpr().getMaxLength());
-
- int length = ptr.getLength();
- int offset = ptr.getOffset();
- byte[] arrayBytes = ptr.get();
-
- getElementExpr().evaluate(tuple, ptr);
-
- checkSizeCompatibility(ptr);
- coerceBytes(ptr);
- return PArrayDataType.prependItemToArray(ptr, length, offset, arrayBytes, getBaseType(), arrayLength, getMaxLength(), getArrayExpr().getSortOrder());
- }
-
- @Override
- public PDataType getDataType() {
- return children.get(1).getDataType();
- }
-
- @Override
- public Integer getMaxLength() {
- return this.children.get(1).getMaxLength();
- }
-
- @Override
- public SortOrder getSortOrder() {
- return getChildren().get(1).getSortOrder();
+ protected boolean modifierFunction(ImmutableBytesWritable ptr, int len, int offset,
+ byte[] arrayBytes, PDataType baseDataType, int arrayLength, Integer maxLength,
+ Expression arrayExp) {
+ return PArrayDataType.prependItemToArray(ptr, len, offset, arrayBytes, baseDataType, arrayLength, getMaxLength(), arrayExp.getSortOrder());
}
@Override
public String getName() {
return NAME;
}
-
- @Override
- public Expression getArrayExpr() {
- return getChildren().get(1);
- }
-
- @Override
- public Expression getElementExpr() {
- return getChildren().get(0);
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/afb21ba1/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
index 86f22f7..4e32cc0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
@@ -21,7 +21,6 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.text.Format;
-import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
@@ -129,6 +128,19 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
return 0;
}
+
+ public static int serializeNulls(byte[] bytes, int position, int nulls){
+ int nMultiplesOver255 = nulls / 255;
+ while (nMultiplesOver255-- > 0) {
+ bytes[position++] = 1;
+ }
+ int nRemainingNulls = nulls % 255;
+ if (nRemainingNulls > 0) {
+ byte nNullByte = SortOrder.invert((byte)(nRemainingNulls-1));
+ bytes[position++] = nNullByte;
+ }
+ return position;
+ }
public static void writeEndSeperatorForVarLengthArray(DataOutputStream oStream) throws IOException {
oStream.write(QueryConstants.SEPARATOR_BYTE);
@@ -246,6 +258,10 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
pArr = new PhoenixArray(pArr, desiredMaxLength);
}
}
+ //Coerce to new max length when only max lengths differ
+ if(actualType == desiredType && !pArr.isPrimitiveType() && maxLength != null && maxLength != desiredMaxLength){
+ pArr = new PhoenixArray(pArr, desiredMaxLength);
+ }
baseType = desiredBaseType;
ptr.set(toBytes(pArr, baseType, expectedModifier));
} else {
@@ -460,6 +476,11 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
public static boolean appendItemToArray(ImmutableBytesWritable ptr, int length, int offset, byte[] arrayBytes, PDataType baseType, int arrayLength, Integer maxLength, SortOrder sortOrder) {
+ if (ptr.getLength() == 0) {
+ ptr.set(arrayBytes, offset, length);
+ return true;
+ }
+
int elementLength = maxLength == null ? ptr.getLength() : maxLength;
//padding
@@ -617,16 +638,8 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
currentPosition++;
newOffsetArrayPosition = offsetArrayPosition + lengthIncrease;
- while (nMultiplesOver255-- > 0) {
- newArray[currentPosition] = (byte) 1;
- currentPosition++;
- }
- // Write a byte for the remaining null elements
- if (nRemainingNulls > 0) {
- byte nNullByte = SortOrder.invert((byte) (nRemainingNulls - 1));
- newArray[currentPosition] = nNullByte; // Single byte for repeating nulls
- currentPosition++;
- }
+ //serialize nulls at the beginning
+ currentPosition = serializeNulls(newArray, currentPosition, nulls);
} else {
if (!useInt) {
if (PArrayDataType.useShortForOffsetArray(endElementPosition)) {
@@ -702,6 +715,134 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
Bytes.putByte(newArray, currentPosition, arrayBytes[offset + length - 1]);
}
+ public static boolean concatArrays(ImmutableBytesWritable ptr, int array1BytesLength, int array1BytesOffset, byte[] array1Bytes, PDataType baseType, int actualLengthOfArray1, int actualLengthOfArray2) {
+ int array2BytesLength = ptr.getLength();
+ int array2BytesOffset = ptr.getOffset();
+ byte[] array2Bytes = ptr.get();
+
+ byte[] newArray;
+
+ if (!baseType.isFixedWidth()) {
+ int offsetArrayPositionArray1 = Bytes.toInt(array1Bytes, array1BytesOffset + array1BytesLength - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT);
+ int offsetArrayPositionArray2 = Bytes.toInt(array2Bytes, array2BytesOffset + array2BytesLength - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT);
+ int offsetArrayLengthArray1 = array1BytesLength - offsetArrayPositionArray1 - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE;
+ int offsetArrayLengthArray2 = array2BytesLength - offsetArrayPositionArray2 - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE;
+ int newArrayLength = actualLengthOfArray1 + actualLengthOfArray2;
+ int nullsAtTheEndOfArray1 = 0;
+ int nullsAtTheBeginningOfArray2 = 0;
+ //checks whether offset array consists of shorts or integers
+ boolean useIntArray1 = offsetArrayLengthArray1 / actualLengthOfArray1 == Bytes.SIZEOF_INT;
+ boolean useIntArray2 = offsetArrayLengthArray2 / actualLengthOfArray2 == Bytes.SIZEOF_INT;
+ boolean useIntNewArray = false;
+ //count nulls at the end of array 1
+ for (int index = actualLengthOfArray1 - 1; index > -1; index--) {
+ int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1);
+ if (array1Bytes[array1BytesOffset + offset] == QueryConstants.SEPARATOR_BYTE) {
+ nullsAtTheEndOfArray1++;
+ } else {
+ break;
+ }
+ }
+ //count nulls at the beginning of the array 2
+ int array2FirstNonNullElementOffset = 0;
+ int array2FirstNonNullIndex = 0;
+ for (int index = 0; index < actualLengthOfArray2; index++) {
+ int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2);
+ if (array2Bytes[array2BytesOffset + offset] == QueryConstants.SEPARATOR_BYTE) {
+ nullsAtTheBeginningOfArray2++;
+ } else {
+ array2FirstNonNullIndex = index;
+ array2FirstNonNullElementOffset = offset;
+ break;
+ }
+ }
+ int nullsInMiddleAfterConcat = nullsAtTheEndOfArray1 + nullsAtTheBeginningOfArray2;
+ int bytesForNullsBefore = nullsAtTheBeginningOfArray2 / 255 + (nullsAtTheBeginningOfArray2 % 255 == 0 ? 0 : 1);
+ int bytesForNullsAfter = nullsInMiddleAfterConcat / 255 + (nullsInMiddleAfterConcat % 255 == 0 ? 0 : 1);
+ //Increase of length required to store nulls
+ int lengthIncreaseForNulls = bytesForNullsAfter - bytesForNullsBefore;
+ //Length increase incremented by one when there were no nulls at the beginning of array and when there are
+ //nulls at the end of array 1 as we need to allocate a byte for separator byte in this case.
+ lengthIncreaseForNulls += nullsAtTheBeginningOfArray2 == 0 && nullsAtTheEndOfArray1 != 0 ? Bytes.SIZEOF_BYTE : 0;
+ int newOffsetArrayPosition = offsetArrayPositionArray1 + offsetArrayPositionArray2 + lengthIncreaseForNulls - 2 * Bytes.SIZEOF_BYTE;
+ int endElementPositionOfArray2 = getOffset(array2Bytes, actualLengthOfArray2 - 1, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2);
+ int newEndElementPosition = lengthIncreaseForNulls + endElementPositionOfArray2 + offsetArrayPositionArray1 - 2 * Bytes.SIZEOF_BYTE;
+ //Creates a byre array to store the concatenated array
+ if (PArrayDataType.useShortForOffsetArray(newEndElementPosition)) {
+ newArray = new byte[newOffsetArrayPosition + newArrayLength * Bytes.SIZEOF_SHORT + Bytes.SIZEOF_INT + Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE];
+ } else {
+ useIntNewArray = true;
+ newArray = new byte[newOffsetArrayPosition + newArrayLength * Bytes.SIZEOF_INT + Bytes.SIZEOF_INT + Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE];
+ }
+
+ int currentPosition = 0;
+ //Copies all the elements from array 1 to new array
+ System.arraycopy(array1Bytes, array1BytesOffset, newArray, currentPosition, offsetArrayPositionArray1 - 2 * Bytes.SIZEOF_BYTE);
+ currentPosition = offsetArrayPositionArray1 - 2 * Bytes.SIZEOF_BYTE;
+ int array2StartingPosition = currentPosition;
+ currentPosition += nullsInMiddleAfterConcat != 0 ? 1 : 0;
+ //Writes nulls in the middle of the array.
+ currentPosition = serializeNulls(newArray, currentPosition, nullsInMiddleAfterConcat);
+ //Copies the elements from array 2 beginning from the first non null element.
+ System.arraycopy(array2Bytes, array2BytesOffset + array2FirstNonNullElementOffset, newArray, currentPosition, offsetArrayPositionArray2 - array2FirstNonNullElementOffset);
+ currentPosition += offsetArrayPositionArray2 - array2FirstNonNullElementOffset;
+
+ //Writing offset arrays
+ if (useIntNewArray) {
+ //offsets for the elements from array 1. Simply copied.
+ for (int index = 0; index < actualLengthOfArray1; index++) {
+ int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1);
+ Bytes.putInt(newArray, currentPosition, offset);
+ currentPosition += Bytes.SIZEOF_INT;
+ }
+ //offsets for nulls in the middle
+ for (int index = 0; index < array2FirstNonNullIndex; index++) {
+ int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2);
+ Bytes.putInt(newArray, currentPosition, offset + array2StartingPosition);
+ currentPosition += Bytes.SIZEOF_INT;
+ }
+ //offsets for the elements from the first non null element from array 2
+ int part2NonNullStartingPosition = array2StartingPosition + bytesForNullsAfter + (bytesForNullsAfter == 0 ? 0 : Bytes.SIZEOF_BYTE);
+ for (int index = array2FirstNonNullIndex; index < actualLengthOfArray2; index++) {
+ int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2);
+ Bytes.putInt(newArray, currentPosition, offset - array2FirstNonNullElementOffset + part2NonNullStartingPosition);
+ currentPosition += Bytes.SIZEOF_INT;
+ }
+ } else {
+ //offsets for the elements from array 1. Simply copied.
+ for (int index = 0; index < actualLengthOfArray1; index++) {
+ int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1);
+ Bytes.putShort(newArray, currentPosition, (short) (offset - Short.MAX_VALUE));
+ currentPosition += Bytes.SIZEOF_SHORT;
+ }
+ //offsets for nulls in the middle
+ for (int index = 0; index < array2FirstNonNullIndex; index++) {
+ int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2);
+ Bytes.putShort(newArray, currentPosition, (short) (offset + array2StartingPosition - Short.MAX_VALUE));
+ currentPosition += Bytes.SIZEOF_SHORT;
+ }
+ //offsets for the elements from the first non null element from array 2
+ int part2NonNullStartingPosition = array2StartingPosition + bytesForNullsAfter + (bytesForNullsAfter == 0 ? 0 : Bytes.SIZEOF_BYTE);
+ for (int index = array2FirstNonNullIndex; index < actualLengthOfArray2; index++) {
+ int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2);
+ Bytes.putShort(newArray, currentPosition, (short) (offset - array2FirstNonNullElementOffset + part2NonNullStartingPosition - Short.MAX_VALUE));
+ currentPosition += Bytes.SIZEOF_SHORT;
+ }
+ }
+ Bytes.putInt(newArray, currentPosition, newOffsetArrayPosition);
+ currentPosition += Bytes.SIZEOF_INT;
+ Bytes.putInt(newArray, currentPosition, useIntNewArray ? -newArrayLength : newArrayLength);
+ currentPosition += Bytes.SIZEOF_INT;
+ Bytes.putByte(newArray, currentPosition, array1Bytes[array1BytesOffset + array1BytesLength - 1]);
+ } else {
+ newArray = new byte[array1BytesLength + array2BytesLength];
+ System.arraycopy(array1Bytes, array1BytesOffset, newArray, 0, array1BytesLength);
+ System.arraycopy(array2Bytes, array2BytesOffset, newArray, array1BytesLength, array2BytesLength);
+ }
+ ptr.set(newArray);
+ return true;
+ }
+
public static int serailizeOffsetArrayIntoStream(DataOutputStream oStream, TrustedByteArrayOutputStream byteStream,
int noOfElements, int maxOffset, int[] offsetPos) throws IOException {
int offsetPosition = (byteStream.size());