You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/03/01 02:46:16 UTC
[2/3] git commit: PHOENIX-37 Prevent deserialize and re-serialize all
array elements in construction of array (ramkrishna)
PHOENIX-37 Prevent deserialize and re-serialize all array elements in construction of array (ramkrishna)
Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/27e00aec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/27e00aec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/27e00aec
Branch: refs/heads/master
Commit: 27e00aec132e45155f648c235f2ea221c7589d99
Parents: c3fa6bf
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Feb 28 17:41:58 2014 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Feb 28 17:45:31 2014 -0800
----------------------------------------------------------------------
.../expression/ArrayConstructorExpression.java | 120 +++-
.../expression/function/ArrayIndexFunction.java | 14 +-
.../apache/phoenix/schema/PArrayDataType.java | 671 ++++++++++++-------
.../org/apache/phoenix/schema/PDataType.java | 56 +-
.../org/apache/phoenix/schema/PhoenixArray.java | 16 +
.../org/apache/phoenix/schema/ValueSchema.java | 4 +-
.../org/apache/phoenix/end2end/ArrayTest.java | 176 ++++-
.../phoenix/schema/PDataTypeForArraysTest.java | 613 ++++++++++++++++-
8 files changed, 1343 insertions(+), 327 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/27e00aec/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
index bf0bdf1..9e9ca0c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
@@ -1,24 +1,17 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by
+ * applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
*/
package org.apache.phoenix.expression;
import java.io.DataInput;
import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.sql.Types;
import java.util.Arrays;
@@ -26,11 +19,11 @@ import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.WritableUtils;
-
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PArrayDataType;
import org.apache.phoenix.schema.PDataType;
-import org.apache.phoenix.schema.PhoenixArray;
import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.util.TrustedByteArrayOutputStream;
/**
* Creates an expression for Upsert with Values/Select using ARRAY
@@ -39,18 +32,31 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
private PDataType baseType;
private int position = -1;
private Object[] elements;
-
-
+ private TrustedByteArrayOutputStream byteStream = null;
+ private DataOutputStream oStream = null;
+ private int estimatedSize = 0;
+ // store the offset postion in this. Later based on the total size move this to a byte[]
+ // and serialize into byte stream
+ private int[] offsetPos;
+
public ArrayConstructorExpression(List<Expression> children, PDataType baseType) {
super(children);
init(baseType);
+ estimatedSize = PArrayDataType.estimateSize(this.children.size(), this.baseType);
+ if (!this.baseType.isFixedWidth()) {
+ offsetPos = new int[children.size()];
+ byteStream = new TrustedByteArrayOutputStream(estimatedSize);
+ } else {
+ byteStream = new TrustedByteArrayOutputStream(estimatedSize);
+ }
+
}
private void init(PDataType baseType) {
this.baseType = baseType;
elements = new Object[getChildren().size()];
}
-
+
@Override
public PDataType getDataType() {
return PDataType.fromTypeId(baseType.getSqlType() + Types.ARRAY);
@@ -62,27 +68,71 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
position = 0;
Arrays.fill(elements, null);
}
-
+
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
- for (int i = position >= 0 ? position : 0; i < elements.length; i++) {
- Expression child = children.get(i);
- if (!child.evaluate(tuple, ptr)) {
- if (tuple != null && !tuple.isImmutable()) {
- if (position >= 0) position = i;
- return false;
+ try {
+ int noOfElements = children.size();
+ int nNulls = 0;
+ oStream = new DataOutputStream(byteStream);
+ for (int i = position >= 0 ? position : 0; i < elements.length; i++) {
+ Expression child = children.get(i);
+ if (!child.evaluate(tuple, ptr)) {
+ if (tuple != null && !tuple.isImmutable()) {
+ if (position >= 0) position = i;
+ return false;
+ }
+ } else {
+ // track the offset position here from the size of the byteStream
+ if (!baseType.isFixedWidth()) {
+ // Any variable length array would follow the below order
+ // Every element would be seperated by a seperator byte '0'
+ // Null elements are counted and once a first non null element appears we
+ // write the count of the nulls prefixed with a seperator byte
+ // Trailing nulls are not taken into account
+ // The last non null element is followed by two seperator bytes
+ // For eg
+ // a, b, null, null, c, null would be
+ // 65 0 66 0 0 2 67 0 0 0
+ // a null null null b c null d would be
+ // 65 0 0 3 66 0 67 0 0 1 68 0 0 0
+ if (ptr.getLength() == 0) {
+ offsetPos[i] = byteStream.size();
+ nNulls++;
+ } else {
+ PArrayDataType.serializeNulls(oStream, nNulls);
+ offsetPos[i] = byteStream.size();
+ oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength());
+ oStream.write(QueryConstants.SEPARATOR_BYTE);
+ }
+ } else { // No nulls for fixed length
+ oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength());
+ }
}
- } else {
- elements[i] = baseType.toObject(ptr, child.getDataType(), child.getSortOrder());
+ }
+ if (position >= 0) position = elements.length;
+ if (!baseType.isFixedWidth()) {
+ // Double seperator byte to show end of the non null array
+ PArrayDataType.writeEndSeperatorForVarLengthArray(oStream);
+ noOfElements = PArrayDataType.serailizeOffsetArrayIntoStream(oStream, byteStream, noOfElements,
+ offsetPos[offsetPos.length - 1], offsetPos);
+ }
+ PArrayDataType.serializeHeaderInfoIntoStream(oStream, noOfElements);
+ ptr.set(byteStream.getBuffer(), 0, byteStream.size());
+ return true;
+ } catch (IOException e) {
+ throw new RuntimeException("Exception while serializing the byte array");
+ } finally {
+ try {
+ byteStream.close();
+ oStream.close();
+ } catch (IOException e) {
+ // Should not happen
}
}
- if (position >= 0) position = elements.length;
- PhoenixArray array = PArrayDataType.instantiatePhoenixArray(baseType, elements);
- // FIXME: Need to see if this creation of an array and again back to byte[] can be avoided
- ptr.set(getDataType().toBytes(array));
- return true;
}
-
+
+
@Override
public void readFields(DataInput input) throws IOException {
super.readFields(input);
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/27e00aec/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java
index 504bdc8..17dce62 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java
@@ -21,7 +21,6 @@ import java.sql.Types;
import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.parse.FunctionParseNode.Argument;
import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
@@ -67,7 +66,7 @@ public class ArrayIndexFunction extends ScalarFunction {
// Given a ptr to the entire array, set ptr to point to a particular element within that array
// given the type of an array element (see comments in PDataTypeForArray)
- PArrayDataType.positionAtArrayElement(ptr, index, getDataType());
+ PArrayDataType.positionAtArrayElement(ptr, index-1, getDataType(), getByteSize());
return true;
}
@@ -77,8 +76,17 @@ public class ArrayIndexFunction extends ScalarFunction {
return PDataType.fromTypeId(children.get(0).getDataType().getSqlType()
- Types.ARRAY);
}
-
+
@Override
+ public Integer getByteSize() {
+ return this.children.get(0).getMaxLength();
+ }
+
+ @Override
+ public Integer getMaxLength() {
+ return this.children.get(0).getMaxLength();
+ }
+ @Override
public String getName() {
return NAME;
}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/27e00aec/phoenix-core/src/main/java/org/apache/phoenix/schema/PArrayDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PArrayDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PArrayDataType.java
index 2cdb04b..047e6f0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PArrayDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PArrayDataType.java
@@ -17,21 +17,33 @@
*/
package org.apache.phoenix.schema;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.sql.Types;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.StringUtil;
+import org.apache.phoenix.util.TrustedByteArrayOutputStream;
/**
- * The datatype for PColummns that are Arrays
+ * The datatype for PColummns that are Arrays. Any variable length array would follow the below order.
+ * Every element would be seperated by a seperator byte '0'. Null elements are counted and once a first
+ * non null element appears we write the count of the nulls prefixed with a seperator byte.
+ * Trailing nulls are not taken into account. The last non null element is followed by two seperator bytes.
+ * For eg a, b, null, null, c, null -> 65 0 66 0 0 2 67 0 0 0
+ * a null null null b c null d -> 65 0 0 3 66 0 67 0 0 1 68 0 0 0.
+ * The reason we use this serialization format is to allow the
+ * byte array of arrays of the same type to be directly comparable against each other.
+ * This prevents a costly deserialization on compare and allows an array column to be used as the last column in a primary key constraint.
*/
public class PArrayDataType {
- private static final int MAX_POSSIBLE_VINT_LENGTH = 2;
- private static final byte ARRAY_SERIALIZATION_VERSION = 1;
+ public static final byte ARRAY_SERIALIZATION_VERSION = 1;
public PArrayDataType() {
}
@@ -39,36 +51,110 @@ public class PArrayDataType {
if(object == null) {
throw new ConstraintViolationException(this + " may not be null");
}
- int size = PDataType.fromTypeId((baseType.getSqlType() + Types.ARRAY)).estimateByteSize(object);
-
+ Pair<Integer, Integer> nullsVsNullRepeationCounter = new Pair<Integer, Integer>();
+ int size = estimateByteSize(object, nullsVsNullRepeationCounter,
+ PDataType.fromTypeId((baseType.getSqlType() + Types.ARRAY)));
int noOfElements = ((PhoenixArray)object).numElements;
if(noOfElements == 0) {
return ByteUtil.EMPTY_BYTE_ARRAY;
}
- ByteBuffer buffer;
- int capacity = 0;
- if (!baseType.isFixedWidth() || baseType.isCoercibleTo(PDataType.VARCHAR)) {
- // variable
- if (calculateMaxOffset(size)) {
- // Use Short to represent the offset
- capacity = initOffsetArray(noOfElements, Bytes.SIZEOF_SHORT);
- } else {
- capacity = initOffsetArray(noOfElements, Bytes.SIZEOF_INT);
- // Negate the number of elements
- noOfElements = -noOfElements;
- }
- buffer = ByteBuffer.allocate(size + capacity + Bytes.SIZEOF_INT+ Bytes.SIZEOF_BYTE);
+ TrustedByteArrayOutputStream byteStream = null;
+ if (!baseType.isFixedWidth()) {
+ size += ((2 * Bytes.SIZEOF_BYTE) + (noOfElements - nullsVsNullRepeationCounter.getFirst()) * Bytes.SIZEOF_BYTE)
+ + (nullsVsNullRepeationCounter.getSecond() * 2 * Bytes.SIZEOF_BYTE);
+ // Assume an offset array that fit into Short.MAX_VALUE. Also not considering nulls that could be > 255
+ // In both of these cases, finally an array copy would happen
+ int capacity = noOfElements * Bytes.SIZEOF_SHORT;
+ // Here the int for noofelements, byte for the version, int for the offsetarray position and 2 bytes for the end seperator
+ byteStream = new TrustedByteArrayOutputStream(size + capacity + Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT);
} else {
- buffer = ByteBuffer.allocate(size);
+ // Here the int for noofelements, byte for the version
+ byteStream = new TrustedByteArrayOutputStream(size + Bytes.SIZEOF_INT+ Bytes.SIZEOF_BYTE);
}
- return bytesFromByteBuffer((PhoenixArray)object, buffer, noOfElements, baseType, capacity);
+ DataOutputStream oStream = new DataOutputStream(byteStream);
+ return createArrayBytes(byteStream, oStream, (PhoenixArray)object, noOfElements, baseType, 0);
}
+
+ private byte[] rewriteArrayBytes(byte[] bytes, Integer desiredMaxLength, Integer maxLength, PDataType baseType) {
+ if (bytes == null || bytes.length == 0) { return null; }
+ ByteBuffer buffer = ByteBuffer.wrap(bytes, 0, bytes.length);
+ int initPos = buffer.position();
+ buffer.position((buffer.limit() - (Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT)));
+ int noOfElemPos = buffer.position();
+ int noOfElements = buffer.getInt();
+ int temp = noOfElements;
+ if (noOfElements < 0) {
+ noOfElements = (-noOfElements);
+ }
+ buffer.position(initPos);
+ ByteBuffer newBuffer = ByteBuffer.allocate(Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT
+ + (noOfElements * desiredMaxLength));
+ for (int i = 0; i < noOfElements; i++) {
+ byte[] val = new byte[(noOfElemPos - initPos)/noOfElements];
+ int length = val.length;
+ buffer.get(val);
+ newBuffer.put(val);
+ while(length < desiredMaxLength) {
+ length++;
+ newBuffer.put(StringUtil.SPACE_UTF8);
+ }
+ }
+ newBuffer.putInt(temp);
+ newBuffer.put(ARRAY_SERIALIZATION_VERSION);
+ return newBuffer.array();
+ }
- private boolean calculateMaxOffset(int size) {
- // If the total size + Offset postion ptr + Numelements in Vint is less than Short
- if ((size + Bytes.SIZEOF_INT + MAX_POSSIBLE_VINT_LENGTH) <= (2 * Short.MAX_VALUE)) {
+ public static int serializeNulls(DataOutputStream oStream, int nulls) throws IOException {
+ // We need to handle 3 different cases here
+ // 1) Arrays with repeating nulls in the middle which is less than 255
+ // 2) Arrays with repeating nulls in the middle which is less than 255 but greater than bytes.MAX_VALUE
+ // 3) Arrays with repeating nulls in the middle greaterh than 255
+ // Take a case where we have two arrays that has the following elements
+ // Array 1 - size : 240, elements = abc, bcd, null, null, bcd,null,null......,null, abc
+ // Array 2 - size : 16 : elements = abc, bcd, null, null, bcd, null, null...null, abc
+ // In both case the elements and the value array will be the same but the Array 1 is actually smaller because it has more nulls.
+ // Now we should have mechanism to show that we treat arrays with more nulls as lesser. Hence in the above case as
+ // 240 > Bytes.MAX_VALUE, by always inverting the number of nulls we would get a +ve value
+ // For Array 2, by inverting we would get a -ve value. On comparison Array 2 > Array 1.
+ // Now for cases where the number of nulls is greater than 255, we would write an those many (byte)1, it is bigger than 255.
+ // This would ensure that we don't compare with triple zero which is used as an end byte
+ if (nulls > 0) {
+ oStream.write(QueryConstants.SEPARATOR_BYTE);
+ int nMultiplesOver255 = nulls / 255;
+ while (nMultiplesOver255-- > 0) {
+ // Don't write a zero byte, as we need to ensure that the only triple zero
+ // byte occurs at the end of the array (i.e. the terminator byte for the
+ // element plus the double zero byte at the end of the array).
+ oStream.write((byte)1);
+ }
+ int nRemainingNulls = nulls % 255; // From 0 to 254
+ // Write a byte for the remaining null elements
+ if (nRemainingNulls > 0) {
+ // Remaining null elements is from 1 to 254.
+ // Subtract one and invert so that more remaining nulls becomes smaller than less
+ // remaining nulls and min byte value is always greater than 1, the repeating value
+ // used for arrays with more than 255 repeating null elements.
+ // The reason we invert is that an array with less null elements has a non
+ // null element sooner than an array with more null elements. Thus, the more
+ // null elements you have, the smaller the array becomes.
+ byte nNullByte = SortOrder.invert((byte)(nRemainingNulls-1));
+ oStream.write(nNullByte); // Single byte for repeating nulls
+ }
+ }
+ return 0;
+ }
+
+ public static void writeEndSeperatorForVarLengthArray(DataOutputStream oStream) throws IOException {
+ oStream.write(QueryConstants.SEPARATOR_BYTE);
+ oStream.write(QueryConstants.SEPARATOR_BYTE);
+ }
+
+ public static boolean useShortForOffsetArray(int maxOffset) {
+ // If the max offset is less than Short.MAX_VALUE then offset array can use short
+ if (maxOffset <= (2 * Short.MAX_VALUE)) {
return true;
}
+ // else offset array can use Int
return false;
}
@@ -77,9 +163,47 @@ public class PArrayDataType {
if (array == null || array.baseType == null) {
return 0;
}
- return PDataType.fromTypeId((array.baseType.getSqlType() + Types.ARRAY)).estimateByteSize(object);
+ return estimateByteSize(object, null, PDataType.fromTypeId((array.baseType.getSqlType() + Types.ARRAY)));
}
+ // Estimates the size of the given array and also calculates the number of nulls and its repetition factor
+ public int estimateByteSize(Object o, Pair<Integer, Integer> nullsVsNullRepeationCounter, PDataType baseType) {
+ if (baseType.isFixedWidth()) { return baseType.getByteSize(); }
+ if (baseType.isArrayType()) {
+ PhoenixArray array = (PhoenixArray)o;
+ int noOfElements = array.numElements;
+ int totalVarSize = 0;
+ int nullsRepeationCounter = 0;
+ int nulls = 0;
+ int totalNulls = 0;
+ for (int i = 0; i < noOfElements; i++) {
+ totalVarSize += array.estimateByteSize(i);
+ if (!PDataType.fromTypeId((baseType.getSqlType() - Types.ARRAY)).isFixedWidth()) {
+ if (array.isNull(i)) {
+ nulls++;
+ } else {
+ if (nulls > 0) {
+ totalNulls += nulls;
+ nulls = 0;
+ nullsRepeationCounter++;
+ }
+ }
+ }
+ }
+ if (nullsVsNullRepeationCounter != null) {
+ if (nulls > 0) {
+ totalNulls += nulls;
+ // do not increment nullsRepeationCounter to identify trailing nulls
+ }
+ nullsVsNullRepeationCounter.setFirst(totalNulls);
+ nullsVsNullRepeationCounter.setSecond(nullsRepeationCounter);
+ }
+ return totalVarSize;
+ }
+ // Non fixed width types must override this
+ throw new UnsupportedOperationException();
+ }
+
public boolean isCoercibleTo(PDataType targetType, Object value) {
return targetType.isCoercibleTo(targetType, value);
}
@@ -100,17 +224,36 @@ public class PArrayDataType {
byte[] b, Integer maxLength, Integer desiredMaxLength,
Integer scale, Integer desiredScale) {
PhoenixArray pArr = (PhoenixArray) value;
- Object[] charArr = (Object[]) pArr.array;
+ Object[] arr = (Object[]) pArr.array;
PDataType baseType = PDataType.fromTypeId(srcType.getSqlType()
- Types.ARRAY);
- for (int i = 0 ; i < charArr.length; i++) {
- if (!baseType.isSizeCompatible(baseType, value, b, maxLength,
+ for (int i = 0 ; i < arr.length; i++) {
+ if (!baseType.isSizeCompatible(baseType, arr[i], b, srcType.getMaxLength(arr[i]),
desiredMaxLength, scale, desiredScale)) {
return false;
}
}
return true;
}
+
+ public byte[] coerceBytes(byte[] b, Object value, PDataType actualType, Integer maxLength, Integer scale,
+ Integer desiredMaxLength, Integer desiredScale) {
+ PhoenixArray pArr = (PhoenixArray) value;
+ Object[] arr = (Object[]) pArr.array;
+ PDataType baseType = PDataType.fromTypeId(actualType.getSqlType()
+ - Types.ARRAY);
+ if (baseType.isFixedWidth()) {
+ boolean createNewArray = false;
+ for (int i = 0; i < arr.length; i++) {
+ if (baseType.getMaxLength(arr[i]) < desiredMaxLength) {
+ createNewArray = true;
+ break;
+ }
+ }
+ if (createNewArray) { return rewriteArrayBytes(b, desiredMaxLength, desiredMaxLength, baseType); }
+ }
+ return b;
+ }
public Object toObject(String value) {
@@ -123,83 +266,77 @@ public class PArrayDataType {
return createPhoenixArray(bytes, offset, length, sortOrder,
baseType);
}
-
- public static void positionAtArrayElement(ImmutableBytesWritable ptr, int arrayIndex, PDataType baseDataType) {
- byte[] bytes = ptr.get();
- int initPos = ptr.getOffset();
- int noOfElements = 0;
- noOfElements = Bytes.toInt(bytes, ptr.getOffset() + Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT);
- int noOFElementsSize = Bytes.SIZEOF_INT;
- if(arrayIndex >= noOfElements) {
- throw new IndexOutOfBoundsException(
- "Invalid index "
- + arrayIndex
- + " specified, greater than the no of elements in the array: "
- + noOfElements);
- }
- boolean useShort = true;
- int baseSize = Bytes.SIZEOF_SHORT;
- if (noOfElements < 0) {
- noOfElements = -noOfElements;
- baseSize = Bytes.SIZEOF_INT;
- useShort = false;
- }
- if (baseDataType.getByteSize() == null) {
- int offset = ptr.getOffset() + noOFElementsSize + Bytes.SIZEOF_BYTE;
- int indexOffset = Bytes.toInt(bytes, offset) + ptr.getOffset();
- int valArrayPostion = offset + Bytes.SIZEOF_INT;
- offset += Bytes.SIZEOF_INT;
- int currOff = 0;
- if (noOfElements > 1) {
- while (offset <= (initPos+ptr.getLength())) {
- int nextOff = 0;
- // Skip those many offsets as given in the arrayIndex
- // If suppose there are 5 elements in the array and the arrayIndex = 3
- // This means we need to read the 4th element of the array
- // So inorder to know the length of the 4th element we will read the offset of 4th element and the offset of 5th element.
- // Subtracting the offset of 5th element and 4th element will give the length of 4th element
- // So we could just skip reading the other elements.
- if(useShort) {
- // If the arrayIndex is already the last element then read the last before one element and the last element
- offset = indexOffset + (Bytes.SIZEOF_SHORT * arrayIndex);
- if (arrayIndex == (noOfElements - 1)) {
- currOff = Bytes.toShort(bytes, offset, baseSize) + Short.MAX_VALUE;
- nextOff = indexOffset;
- offset += baseSize;
- } else {
- currOff = Bytes.toShort(bytes, offset, baseSize) + Short.MAX_VALUE;
- offset += baseSize;
- nextOff = Bytes.toShort(bytes, offset, baseSize) + Short.MAX_VALUE;
- offset += baseSize;
- }
- } else {
- // If the arrayIndex is already the last element then read the last before one element and the last element
- offset = indexOffset + (Bytes.SIZEOF_INT * arrayIndex);
- if (arrayIndex == (noOfElements - 1)) {
- currOff = Bytes.toInt(bytes, offset, baseSize);
- nextOff = indexOffset;
- offset += baseSize;
- } else {
- currOff = Bytes.toInt(bytes, offset, baseSize);
- offset += baseSize;
- nextOff = Bytes.toInt(bytes, offset, baseSize);
- offset += baseSize;
- }
- }
- int elementLength = nextOff - currOff;
- ptr.set(bytes, currOff + initPos, elementLength);
- break;
- }
- } else {
- ptr.set(bytes, valArrayPostion + initPos, indexOffset - valArrayPostion);
- }
- } else {
- ptr.set(bytes,
- ptr.getOffset() + arrayIndex * baseDataType.getByteSize()
- + noOFElementsSize + Bytes.SIZEOF_BYTE, baseDataType.getByteSize());
- }
- }
+ public static void positionAtArrayElement(ImmutableBytesWritable ptr, int arrayIndex, PDataType baseDataType,
+ Integer byteSize) {
+ byte[] bytes = ptr.get();
+ int initPos = ptr.getOffset();
+ int noOfElements = 0;
+ noOfElements = Bytes.toInt(bytes, (ptr.getOffset() + ptr.getLength() - (Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT)),
+ Bytes.SIZEOF_INT);
+ if (arrayIndex >= noOfElements) {
+ ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
+ return;
+ }
+ boolean useShort = true;
+ if (noOfElements < 0) {
+ noOfElements = -noOfElements;
+ useShort = false;
+ }
+
+ if (!baseDataType.isFixedWidth()) {
+ int indexOffset = Bytes.toInt(bytes,
+ (ptr.getOffset() + ptr.getLength() - (Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT))) + ptr.getOffset();
+ if(arrayIndex >= noOfElements) {
+ ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
+ } else {
+ // Skip those many offsets as given in the arrayIndex
+ // If suppose there are 5 elements in the array and the arrayIndex = 3
+ // This means we need to read the 4th element of the array
+ // So inorder to know the length of the 4th element we will read the offset of 4th element and the
+ // offset of 5th element.
+ // Subtracting the offset of 5th element and 4th element will give the length of 4th element
+ // So we could just skip reading the other elements.
+ int currOffset = getOffset(bytes, arrayIndex, useShort, indexOffset);
+ int elementLength = 0;
+ if (arrayIndex == (noOfElements - 1)) {
+ elementLength = bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE ? 0 : indexOffset
+ - currOffset - 3;
+ } else {
+ elementLength = bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE ? 0 : getOffset(bytes,
+ arrayIndex + 1, useShort, indexOffset) - currOffset - 1;
+ }
+ ptr.set(bytes, currOffset + initPos, elementLength);
+ }
+ } else {
+ if (byteSize != null) {
+ ptr.set(bytes, ptr.getOffset() + (arrayIndex * byteSize), byteSize);
+ } else {
+ ptr.set(bytes, ptr.getOffset() + (arrayIndex * baseDataType.getByteSize()), baseDataType.getByteSize());
+ }
+ }
+ }
+
+ private static int getOffset(byte[] bytes, int arrayIndex, boolean useShort, int indexOffset) {
+ int offset;
+ if (useShort) {
+ offset = indexOffset + (Bytes.SIZEOF_SHORT * arrayIndex);
+ return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT) + Short.MAX_VALUE;
+ } else {
+ offset = indexOffset + (Bytes.SIZEOF_INT * arrayIndex);
+ return Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
+ }
+ }
+
+ private static int getOffset(ByteBuffer indexBuffer, int arrayIndex, boolean useShort, int indexOffset ) {
+ int offset;
+ if(useShort) {
+ offset = indexBuffer.getShort() + Short.MAX_VALUE;
+ } else {
+ offset = indexBuffer.getInt();
+ }
+ return offset;
+ }
public Object toObject(byte[] bytes, int offset, int length, PDataType baseType) {
return toObject(bytes, offset, length, baseType, SortOrder.getDefault());
@@ -214,150 +351,207 @@ public class PArrayDataType {
return toObject(object, actualType);
}
- // Making this private
/**
- * The format of the byte buffer looks like this for variable length array elements
- * <noofelements><Offset of the index array><elements><offset array>
- * where <noOfelements> - vint
- * <offset of the index array> - int
- * <elements> - these are the values
- * <offset array> - offset of every element written as INT/SHORT
- *
+ * ser
+ * @param byteStream
+ * @param oStream
* @param array
- * @param buffer
* @param noOfElements
- * @param byteSize
- * @param capacity
+ * @param baseType
+ * @param maxLength
+ * @param capacity
* @return
*/
- private byte[] bytesFromByteBuffer(PhoenixArray array, ByteBuffer buffer,
- int noOfElements, PDataType baseType, int capacity) {
- int temp = noOfElements;
- if (buffer == null) return null;
- buffer.put(ARRAY_SERIALIZATION_VERSION);
- buffer.putInt(noOfElements);
- if (!baseType.isFixedWidth() || baseType.isCoercibleTo(PDataType.VARCHAR)) {
- int fillerForOffsetByteArray = buffer.position();
- buffer.position(fillerForOffsetByteArray + Bytes.SIZEOF_INT);
- ByteBuffer offsetArray = ByteBuffer.allocate(capacity);
- if(noOfElements < 0){
- noOfElements = -noOfElements;
+ private byte[] createArrayBytes(TrustedByteArrayOutputStream byteStream, DataOutputStream oStream,
+ PhoenixArray array, int noOfElements, PDataType baseType, Integer maxLength) {
+ try {
+ if (!baseType.isFixedWidth()) {
+ int[] offsetPos = new int[noOfElements];
+ int nulls = 0;
+ for (int i = 0; i < noOfElements; i++) {
+ byte[] bytes = array.toBytes(i);
+ if (bytes.length == 0) {
+ offsetPos[i] = byteStream.size();
+ nulls++;
+ } else {
+ nulls = serializeNulls(oStream, nulls);
+ offsetPos[i] = byteStream.size();
+ oStream.write(bytes, 0, bytes.length);
+ oStream.write(QueryConstants.SEPARATOR_BYTE);
+ }
+ }
+ // Double seperator byte to show end of the non null array
+ PArrayDataType.writeEndSeperatorForVarLengthArray(oStream);
+ noOfElements = PArrayDataType.serailizeOffsetArrayIntoStream(oStream, byteStream, noOfElements,
+ offsetPos[offsetPos.length - 1], offsetPos);
+ } else {
+ for (int i = 0; i < noOfElements; i++) {
+ byte[] bytes = array.toBytes(i);
+ int length = bytes.length;
+ if(baseType == PDataType.CHAR) {
+ oStream.write(bytes, 0, length);
+ while(length <= maxLength) {
+ oStream.writeByte(StringUtil.SPACE_UTF8);
+ length++;
+ }
+ } else {
+ oStream.write(bytes, 0, length);
+ }
+ }
}
- for (int i = 0; i < noOfElements; i++) {
- // Not fixed width
- if (temp < 0) {
- offsetArray.putInt(buffer.position());
- } else {
- offsetArray.putShort((short)(buffer.position() - Short.MAX_VALUE));
- }
- byte[] bytes = array.toBytes(i);
- buffer.put(bytes);
+ serializeHeaderInfoIntoStream(oStream, noOfElements);
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ ptr.set(byteStream.getBuffer(), 0, byteStream.size());
+ return ByteUtil.copyKeyBytesIfNecessary(ptr);
+ } catch (IOException e) {
+ try {
+ byteStream.close();
+ oStream.close();
+ } catch (IOException ioe) {
+
}
- int offsetArrayPosition = buffer.position();
- buffer.put(offsetArray.array());
- buffer.position(fillerForOffsetByteArray);
- buffer.putInt(offsetArrayPosition);
+ }
+ // This should not happen
+ return null;
+ }
+
+ public static int serailizeOffsetArrayIntoStream(DataOutputStream oStream, TrustedByteArrayOutputStream byteStream,
+ int noOfElements, int maxOffset, int[] offsetPos) throws IOException {
+ int offsetPosition = (byteStream.size());
+ byte[] offsetArr = null;
+ boolean useInt = true;
+ if (PArrayDataType.useShortForOffsetArray(maxOffset)) {
+ offsetArr = new byte[PArrayDataType.initOffsetArray(noOfElements, Bytes.SIZEOF_SHORT)];
+ useInt = false;
} else {
- for (int i = 0; i < noOfElements; i++) {
- byte[] bytes = array.toBytes(i);
- buffer.put(bytes);
+ offsetArr = new byte[PArrayDataType.initOffsetArray(noOfElements, Bytes.SIZEOF_INT)];
+ noOfElements = -noOfElements;
+ }
+ int off = 0;
+ if(useInt) {
+ for (int pos : offsetPos) {
+ Bytes.putInt(offsetArr, off, pos);
+ off += Bytes.SIZEOF_INT;
+ }
+ } else {
+ for (int pos : offsetPos) {
+ Bytes.putShort(offsetArr, off, (short)(pos - Short.MAX_VALUE));
+ off += Bytes.SIZEOF_SHORT;
}
}
- return buffer.array();
- }
+ oStream.write(offsetArr);
+ oStream.writeInt(offsetPosition);
+ return noOfElements;
+ }
+
+ public static void serializeHeaderInfoIntoBuffer(ByteBuffer buffer, int noOfElements) {
+ // No of elements
+ buffer.putInt(noOfElements);
+ // Version of the array
+ buffer.put(ARRAY_SERIALIZATION_VERSION);
+ }
- private static int initOffsetArray(int noOfElements, int baseSize) {
+ public static void serializeHeaderInfoIntoStream(DataOutputStream oStream, int noOfElements) throws IOException {
+ // No of elements
+ oStream.writeInt(noOfElements);
+ // Version of the array
+ oStream.write(ARRAY_SERIALIZATION_VERSION);
+ }
+
+ public static int initOffsetArray(int noOfElements, int baseSize) {
// for now create an offset array equal to the noofelements
return noOfElements * baseSize;
}
- private Object createPhoenixArray(byte[] bytes, int offset, int length,
- SortOrder sortOrder, PDataType baseDataType) {
- if(bytes == null || bytes.length == 0) {
- return null;
- }
- ByteBuffer buffer = ByteBuffer.wrap(bytes, offset, length);
- int initPos = buffer.position();
- buffer.get();
- int noOfElements = buffer.getInt();
- boolean useShort = true;
- int baseSize = Bytes.SIZEOF_SHORT;
- if(noOfElements < 0) {
- noOfElements = -noOfElements;
- baseSize = Bytes.SIZEOF_INT;
- useShort = false;
- }
- Object[] elements = (Object[]) java.lang.reflect.Array.newInstance(
- baseDataType.getJavaClass(), noOfElements);
- if (!baseDataType.isFixedWidth() || baseDataType.isCoercibleTo(PDataType.VARCHAR)) {
- int indexOffset = buffer.getInt();
- int valArrayPostion = buffer.position();
- buffer.position(indexOffset + initPos);
- ByteBuffer indexArr = ByteBuffer
- .allocate(initOffsetArray(noOfElements, baseSize));
- byte[] array = indexArr.array();
- buffer.get(array);
- int countOfElementsRead = 0;
- int i = 0;
- int currOff = -1;
- int nextOff = -1;
- if (noOfElements > 1) {
- while (indexArr.hasRemaining()) {
- if (countOfElementsRead < noOfElements) {
- if (currOff == -1) {
- if ((indexArr.position() + 2 * baseSize) <= indexArr
- .capacity()) {
- if (useShort) {
- currOff = indexArr.getShort() + Short.MAX_VALUE;
- nextOff = indexArr.getShort() + Short.MAX_VALUE;
- } else {
- currOff = indexArr.getInt();
- nextOff = indexArr.getInt();
- }
- countOfElementsRead += 2;
- }
- } else {
- currOff = nextOff;
- if(useShort) {
- nextOff = indexArr.getShort() + Short.MAX_VALUE;
- } else {
- nextOff = indexArr.getInt();
- }
- countOfElementsRead += 1;
- }
- int elementLength = nextOff - currOff;
- buffer.position(currOff + initPos);
- byte[] val = new byte[elementLength];
- buffer.get(val);
- elements[i++] = baseDataType.toObject(val,
- sortOrder);
- }
- }
- buffer.position(nextOff + initPos);
- byte[] val = new byte[indexOffset - nextOff];
- buffer.get(val);
- elements[i++] = baseDataType.toObject(val, sortOrder);
- } else {
- byte[] val = new byte[indexOffset - valArrayPostion];
- buffer.position(valArrayPostion + initPos);
- buffer.get(val);
- elements[i++] = baseDataType.toObject(val, sortOrder);
- }
- } else {
- for (int i = 0; i < noOfElements; i++) {
- byte[] val;
- if (baseDataType.getByteSize() == null) {
- val = new byte[length];
- } else {
- val = new byte[baseDataType.getByteSize()];
- }
- buffer.get(val);
- elements[i] = baseDataType.toObject(val, sortOrder);
- }
- }
- return PArrayDataType
- .instantiatePhoenixArray(baseDataType, elements);
- }
+ // Any variable length array would follow the below order
+ // Every element would be seperated by a seperator byte '0'
+ // Null elements are counted and once a first non null element appears we
+ // write the count of the nulls prefixed with a seperator byte
+ // Trailing nulls are not taken into account
+ // The last non null element is followed by two seperator bytes
+ // For eg
+ // a, b, null, null, c, null would be
+ // 65 0 66 0 0 2 67 0 0 0
+ // a null null null b c null d would be
+ // 65 0 0 3 66 0 67 0 0 1 68 0 0 0
+ // Follow the above example to understand how this works
+ private Object createPhoenixArray(byte[] bytes, int offset, int length, SortOrder sortOrder, PDataType baseDataType) {
+ if (bytes == null || bytes.length == 0) { return null; }
+ ByteBuffer buffer = ByteBuffer.wrap(bytes, offset, length);
+ int initPos = buffer.position();
+ buffer.position((buffer.limit() - (Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT)));
+ int noOfElemPos = buffer.position();
+ int noOfElements = buffer.getInt();
+ boolean useShort = true;
+ int baseSize = Bytes.SIZEOF_SHORT;
+ if (noOfElements < 0) {
+ noOfElements = -noOfElements;
+ baseSize = Bytes.SIZEOF_INT;
+ useShort = false;
+ }
+ Object[] elements = (Object[])java.lang.reflect.Array.newInstance(baseDataType.getJavaClass(), noOfElements);
+ if (!baseDataType.isFixedWidth()) {
+ buffer.position(buffer.limit() - (Bytes.SIZEOF_BYTE + (2 * Bytes.SIZEOF_INT)));
+ int indexOffset = buffer.getInt();
+ buffer.position(initPos);
+ buffer.position(indexOffset + initPos);
+ ByteBuffer indexArr = ByteBuffer.allocate(initOffsetArray(noOfElements, baseSize));
+ byte[] array = indexArr.array();
+ buffer.get(array);
+ int countOfElementsRead = 0;
+ int i = 0;
+ int currOffset = -1;
+ int nextOff = -1;
+ boolean foundNull = false;
+ if (noOfElements != 0) {
+ while (countOfElementsRead <= noOfElements) {
+ if (countOfElementsRead == 0) {
+ currOffset = getOffset(indexArr, countOfElementsRead, useShort, indexOffset);
+ countOfElementsRead++;
+ } else {
+ currOffset = nextOff;
+ }
+ if (countOfElementsRead == noOfElements) {
+ nextOff = indexOffset - 2;
+ } else {
+ nextOff = getOffset(indexArr, countOfElementsRead + 1, useShort, indexOffset);
+ }
+ countOfElementsRead++;
+ if ((bytes[currOffset + initPos] != QueryConstants.SEPARATOR_BYTE) && foundNull) {
+ // Found a non null element
+ foundNull = false;
+ }
+ if (bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE) {
+ // Null element
+ foundNull = true;
+ i++;
+ continue;
+ }
+ int elementLength = nextOff - currOffset;
+ buffer.position(currOffset + initPos);
+ // Subtract the seperator from the element length
+ byte[] val = new byte[elementLength - 1];
+ buffer.get(val);
+ elements[i++] = baseDataType.toObject(val, sortOrder);
+ }
+ }
+ } else {
+ buffer.position(initPos);
+ for (int i = 0; i < noOfElements; i++) {
+ byte[] val;
+ if (baseDataType == PDataType.CHAR) {
+ int maxLength = (noOfElemPos - initPos)/noOfElements;
+ // Should be char array
+ val = new byte[maxLength];
+ } else {
+ val = new byte[baseDataType.getByteSize()];
+ }
+ buffer.get(val);
+ elements[i] = baseDataType.toObject(val, sortOrder);
+ }
+ }
+ return PArrayDataType.instantiatePhoenixArray(baseDataType, elements);
+ }
public static PhoenixArray instantiatePhoenixArray(PDataType actualType, Object[] elements) {
return PDataType.instantiatePhoenixArray(actualType, elements);
@@ -378,7 +572,16 @@ public class PArrayDataType {
if(baseType.isFixedWidth()) {
return ((ptr.getLength() - (Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT))/baseType.getByteSize());
}
- return Bytes.toInt(bytes, ptr.getOffset() + Bytes.SIZEOF_BYTE);
+ return Bytes.toInt(bytes, (ptr.getOffset() + ptr.getLength() - (Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT)));
}
+ public static int estimateSize(int size, PDataType baseType) {
+ if(baseType.isFixedWidth()) {
+ return baseType.getByteSize() * size;
+ } else {
+ return size * ValueSchema.ESTIMATED_VARIABLE_LENGTH_SIZE;
+ }
+
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/27e00aec/phoenix-core/src/main/java/org/apache/phoenix/schema/PDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PDataType.java
index 5fc9d87..26fa3eb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PDataType.java
@@ -3432,6 +3432,13 @@ public enum PDataType {
}
@Override
+ public Integer getMaxLength(Object o) {
+ if (o == null) { return null; }
+ byte[] value = (byte[])o;
+ return value.length;
+ }
+
+ @Override
public Object toObject(String value) {
if (value == null || value.length() == 0) {
return null;
@@ -3653,12 +3660,19 @@ public enum PDataType {
return true;
}
- @Override
- public boolean isSizeCompatible(PDataType srcType, Object value,
- byte[] b, Integer maxLength, Integer desiredMaxLength,
- Integer scale, Integer desiredScale) {
- return pDataTypeForArray.isSizeCompatible(srcType, value, b,
- maxLength, desiredMaxLength, scale, desiredScale); }
+ @Override
+ public boolean isSizeCompatible(PDataType srcType, Object value, byte[] b, Integer maxLength,
+ Integer desiredMaxLength, Integer scale, Integer desiredScale) {
+ return pDataTypeForArray.isSizeCompatible(srcType, value, b, maxLength, desiredMaxLength, scale,
+ desiredScale);
+ }
+
+ @Override
+ public byte[] coerceBytes(byte[] b, Object object, PDataType actualType, Integer maxLength, Integer scale,
+ Integer desiredMaxLength, Integer desiredScale) {
+ return pDataTypeForArray.coerceBytes(b, object, actualType, maxLength, scale, desiredMaxLength,
+ desiredScale);
+ }
},
VARBINARY_ARRAY("VARBINARY_ARRAY", Types.ARRAY + PDataType.VARBINARY.getSqlType(), PhoenixArray.class, null) {
@@ -3735,7 +3749,13 @@ public enum PDataType {
return pDataTypeForArray.isSizeCompatible(srcType, value, b,
maxLength, desiredMaxLength, scale, desiredScale);
}
-
+
+ @Override
+ public byte[] coerceBytes(byte[] b, Object object, PDataType actualType, Integer maxLength, Integer scale,
+ Integer desiredMaxLength, Integer desiredScale) {
+ return pDataTypeForArray.coerceBytes(b, object, actualType, maxLength, scale, desiredMaxLength,
+ desiredScale);
+ }
},
BINARY_ARRAY("BINARY_ARRAY", Types.ARRAY + PDataType.BINARY.getSqlType(), PhoenixArray.class, null) {
@Override
@@ -3811,6 +3831,13 @@ public enum PDataType {
return pDataTypeForArray.isSizeCompatible(srcType, value, b,
maxLength, desiredMaxLength, scale, desiredScale);
}
+
+ @Override
+ public byte[] coerceBytes(byte[] b, Object object, PDataType actualType, Integer maxLength, Integer scale,
+ Integer desiredMaxLength, Integer desiredScale) {
+ return pDataTypeForArray.coerceBytes(b, object, actualType, maxLength, scale, desiredMaxLength,
+ desiredScale);
+ }
},
CHAR_ARRAY("CHAR_ARRAY", Types.ARRAY + PDataType.CHAR.getSqlType(), PhoenixArray.class, null) {
@@ -3889,6 +3916,12 @@ public enum PDataType {
maxLength, desiredMaxLength, scale, desiredScale);
}
+ @Override
+ public byte[] coerceBytes(byte[] b, Object object, PDataType actualType, Integer maxLength, Integer scale,
+ Integer desiredMaxLength, Integer desiredScale) {
+ return pDataTypeForArray.coerceBytes(b, object, actualType, maxLength, scale, desiredMaxLength, desiredScale);
+ }
+
},
LONG_ARRAY("LONG_ARRAY", Types.ARRAY + PDataType.LONG.getSqlType(), PhoenixArray.class, null) {
@Override
@@ -5170,18 +5203,11 @@ public enum PDataType {
if(isArrayType()) {
PhoenixArray array = (PhoenixArray)o;
int noOfElements = array.numElements;
- //int vIntSize = WritableUtils.getVIntSize(noOfElements);
- int vIntSize = Bytes.SIZEOF_INT;
-
int totalVarSize = 0;
for (int i = 0; i < noOfElements; i++) {
totalVarSize += array.estimateByteSize(i);
}
- /**
- * Calculate the totalVarSize including the vInt and byte needed for the
- * number of elements and version respectively
- */
- return vIntSize + (totalVarSize) + Bytes.SIZEOF_BYTE;
+ return totalVarSize;
}
// Non fixed width types must override this
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/27e00aec/phoenix-core/src/main/java/org/apache/phoenix/schema/PhoenixArray.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PhoenixArray.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PhoenixArray.java
index 6d46fa7..c44bf36 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PhoenixArray.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PhoenixArray.java
@@ -32,6 +32,7 @@ public class PhoenixArray implements Array,SQLCloseable {
PDataType baseType;
Object array;
int numElements;
+ Integer maxLength;
public PhoenixArray() {
// empty constructor
}
@@ -136,13 +137,28 @@ public class PhoenixArray implements Array,SQLCloseable {
}
public int estimateByteSize(int pos) {
+ if(((Object[])array)[pos] == null) {
+ return 0;
+ }
return this.baseType.estimateByteSize(((Object[])array)[pos]);
}
+ public Integer getMaxLength(int pos) {
+ return this.baseType.getMaxLength(((Object[])array)[pos]);
+ }
+
public byte[] toBytes(int pos) {
return this.baseType.toBytes(((Object[])array)[pos]);
}
+ public boolean isNull(int pos) {
+ if(this.baseType.toBytes(((Object[])array)[pos]).length == 0) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
@Override
public boolean equals(Object obj) {
if (this.numElements != ((PhoenixArray) obj).numElements) {
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/27e00aec/phoenix-core/src/main/java/org/apache/phoenix/schema/ValueSchema.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ValueSchema.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ValueSchema.java
index 8e13fd7..95c99e0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/ValueSchema.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ValueSchema.java
@@ -181,7 +181,9 @@ public abstract class ValueSchema implements Writable {
this.count = count;
this.isNullable = isNullable;
if (this.type.isFixedWidth() && this.type.getByteSize() == null) {
- this.byteSize = datum.getByteSize();
+ if (datum.getByteSize() != null) {
+ this.byteSize = datum.getByteSize();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/27e00aec/phoenix-core/src/test/java/org/apache/phoenix/end2end/ArrayTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/end2end/ArrayTest.java b/phoenix-core/src/test/java/org/apache/phoenix/end2end/ArrayTest.java
index 4394e03..c78abf4 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/end2end/ArrayTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/end2end/ArrayTest.java
@@ -205,7 +205,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT ARRAY_ELEM(a_double_array,1) FROM table_with_array";
+ String query = "SELECT ARRAY_ELEM(a_double_array,2) FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -233,7 +233,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT a_double_array[2] FROM table_with_array";
+ String query = "SELECT a_double_array[3] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -260,7 +260,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT CASE WHEN A_INTEGER = 1 THEN a_double_array ELSE null END [2] FROM table_with_array";
+ String query = "SELECT CASE WHEN A_INTEGER = 1 THEN a_double_array ELSE null END [3] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -302,7 +302,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2
conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
- query = "SELECT ARRAY_ELEM(a_double_array,1) FROM table_with_array";
+ query = "SELECT ARRAY_ELEM(a_double_array,2) FROM table_with_array";
statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -332,7 +332,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
String query = "upsert into table_with_array(ORGANIZATION_ID,ENTITY_ID,a_double_array) "
+ "SELECT organization_id, entity_id, a_double_array FROM " + SIMPLE_TABLE_WITH_ARRAY
- + " WHERE a_double_array[1] = 89.96d";
+ + " WHERE a_double_array[2] = 89.96d";
PreparedStatement statement = conn.prepareStatement(query);
int executeUpdate = statement.executeUpdate();
assertEquals(1, executeUpdate);
@@ -343,7 +343,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 4));
conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
- query = "SELECT ARRAY_ELEM(a_double_array,1) FROM table_with_array";
+ query = "SELECT ARRAY_ELEM(a_double_array,2) FROM table_with_array";
statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -393,6 +393,71 @@ public class ArrayTest extends BaseClientManagedTimeTest {
}
@Test
+ public void testSelectWithArrayWithColumnRefWithVarLengthArray() throws Exception {
+ long ts = nextTimestamp();
+ String tenantId = getOrganizationId();
+ createTableWithArray(BaseConnectedQueryTest.getUrl(), getDefaultSplits(tenantId), null, ts - 2);
+ initTablesWithArrays(tenantId, null, ts, false);
+ String query = "SELECT b_string,ARRAY['abc','defgh',b_string] FROM table_with_array where organization_id = '"
+ + tenantId + "'";
+ Properties props = new Properties(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2
+ Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
+ try {
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ String val = rs.getString(1);
+ assertEquals(val, "b");
+ Array array = rs.getArray(2);
+ // Need to support primitive
+ String[] strArr = new String[3];
+ strArr[0] = "abc";
+ strArr[1] = "defgh";
+ strArr[2] = "b";
+ Array resultArr = conn.createArrayOf("VARCHAR", strArr);
+ assertEquals(resultArr, array);
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testSelectWithArrayWithColumnRefWithVarLengthArrayWithNullValue() throws Exception {
+ long ts = nextTimestamp();
+ String tenantId = getOrganizationId();
+ createTableWithArray(BaseConnectedQueryTest.getUrl(), getDefaultSplits(tenantId), null, ts - 2);
+ initTablesWithArrays(tenantId, null, ts, false);
+ String query = "SELECT b_string,ARRAY['abc',null,'bcd',null,null,b_string] FROM table_with_array where organization_id = '"
+ + tenantId + "'";
+ Properties props = new Properties(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2
+ Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
+ try {
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ String val = rs.getString(1);
+ assertEquals(val, "b");
+ Array array = rs.getArray(2);
+ // Need to support primitive
+ String[] strArr = new String[6];
+ strArr[0] = "abc";
+ strArr[1] = null;
+ strArr[2] = "bcd";
+ strArr[3] = null;
+ strArr[4] = null;
+ strArr[5] = "b";
+ Array resultArr = conn.createArrayOf("VARCHAR", strArr);
+ assertEquals(resultArr, array);
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
public void testUpsertSelectWithColumnRef() throws Exception {
long ts = nextTimestamp();
String tenantId = getOrganizationId();
@@ -406,7 +471,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
String query = "upsert into table_with_array(ORGANIZATION_ID,ENTITY_ID, a_unsigned_double, a_double_array) "
+ "SELECT organization_id, entity_id, x_double, ARRAY[23.4d, 22.1d, x_double] FROM " + SIMPLE_TABLE_WITH_ARRAY
- + " WHERE a_double_array[1] = 89.96d";
+ + " WHERE a_double_array[2] = 89.96d";
PreparedStatement statement = conn.prepareStatement(query);
int executeUpdate = statement.executeUpdate();
assertEquals(1, executeUpdate);
@@ -417,7 +482,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 4));
conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
- query = "SELECT ARRAY_ELEM(a_double_array,1) FROM table_with_array";
+ query = "SELECT ARRAY_ELEM(a_double_array,2) FROM table_with_array";
statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -435,6 +500,70 @@ public class ArrayTest extends BaseClientManagedTimeTest {
}
}
+ @Test
+ public void testCharArraySpecificIndex() throws Exception {
+ long ts = nextTimestamp();
+ String tenantId = getOrganizationId();
+ createSimpleTableWithArray(BaseConnectedQueryTest.getUrl(), getDefaultSplits(tenantId), null, ts - 2);
+ initSimpleArrayTable(tenantId, null, ts, false);
+ String query = "SELECT a_char_array[2] FROM SIMPLE_TABLE_WITH_ARRAY";
+ Properties props = new Properties(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2
+ Connection conn = DriverManager.getConnection(PHOENIX_JDBC_URL, props);
+ try {
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ String[] charArr = new String[1];
+ charArr[0] = "b";
+ String result = rs.getString(1);
+ assertEquals(charArr[0], result);
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testFixedWidthCharArray() throws Exception {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ long ts = nextTimestamp();
+ Properties props = new Properties(TEST_PROPERTIES);
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.createStatement().execute("CREATE TABLE t ( k VARCHAR PRIMARY KEY, a CHAR(5) ARRAY)");
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.getMetaData().getColumns(null, null, "T", "A");
+ assertTrue(rs.next());
+ assertEquals(5, rs.getInt("COLUMN_SIZE"));
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(getUrl(), props);
+ stmt = conn.prepareStatement("UPSERT INTO t VALUES(?,?)");
+ stmt.setString(1, "a");
+ String[] s = new String[] {"1","2"};
+ Array array = conn.createArrayOf("CHAR", s);
+ stmt.setArray(2, array);
+ stmt.execute();
+ conn.commit();
+ conn.close();
+
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 40));
+ conn = DriverManager.getConnection(getUrl(), props);
+ rs = conn.createStatement().executeQuery("SELECT k, a[2] FROM t");
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals("2",rs.getString(2));
+ conn.close();
+
+ }
+
@Test
public void testSelectArrayUsingUpsertLikeSyntax() throws Exception {
long ts = nextTimestamp();
@@ -473,7 +602,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
int a_index = 0;
- String query = "SELECT a_double_array[1] FROM table_with_array where a_double_array["+a_index+"1]<?";
+ String query = "SELECT a_double_array[2] FROM table_with_array where a_double_array["+a_index+"2]<?";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -504,7 +633,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT a_double_array[1] FROM table_with_array GROUP BY a_double_array[1]";
+ String query = "SELECT a_double_array[2] FROM table_with_array GROUP BY a_double_array[2]";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -533,7 +662,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, true);
- String query = "SELECT a_string_array[1] FROM table_with_array";
+ String query = "SELECT a_string_array[2] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -558,7 +687,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT a_string_array[2],A_INTEGER FROM table_with_array";
+ String query = "SELECT a_string_array[3],A_INTEGER FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -586,7 +715,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT A_INTEGER, a_string_array[2] FROM table_with_array";
+ String query = "SELECT A_INTEGER, a_string_array[3] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -614,7 +743,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT a_string_array[2], a_double_array[1] FROM table_with_array";
+ String query = "SELECT a_string_array[3], a_double_array[2] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -644,7 +773,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT a_string_array[0], a_string_array[2] FROM table_with_array";
+ String query = "SELECT a_string_array[1], a_string_array[3] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -673,7 +802,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT a_string_array[2], a_string_array[2] FROM table_with_array";
+ String query = "SELECT a_string_array[3], a_string_array[3] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -701,7 +830,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
createTableWithArray(BaseConnectedQueryTest.getUrl(),
getDefaultSplits(tenantId), null, ts - 2);
initTablesWithArrays(tenantId, null, ts, false);
- String query = "SELECT a_string_array[2] FROM table_with_array";
+ String query = "SELECT a_string_array[3] FROM table_with_array";
Properties props = new Properties(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
@@ -743,7 +872,6 @@ public class ArrayTest extends BaseClientManagedTimeTest {
assertEquals(resultArray, array);
Assert.fail("Should have failed");
} catch (Exception e) {
- System.out.println("");
} finally {
conn.close();
}
@@ -863,6 +991,7 @@ public class ArrayTest extends BaseClientManagedTimeTest {
+ " entity_id char(15) not null,\n"
+ " x_double double,\n"
+ " a_double_array double array[],\n"
+ + " a_char_array char(5) array[],\n"
+ " CONSTRAINT pk PRIMARY KEY (organization_id, entity_id)\n"
+ ")";
BaseTest.createTestTable(url, ddlStmt, bs, ts);
@@ -882,8 +1011,8 @@ public class ArrayTest extends BaseClientManagedTimeTest {
" ORGANIZATION_ID, " +
" ENTITY_ID, " +
" x_double, " +
- " a_double_array)" +
- "VALUES (?, ?, ?, ?)");
+ " a_double_array, a_char_array)" +
+ "VALUES (?, ?, ?, ?, ?)");
stmt.setString(1, tenantId);
stmt.setString(2, ROW1);
stmt.setDouble(3, 1.2d);
@@ -894,6 +1023,13 @@ public class ArrayTest extends BaseClientManagedTimeTest {
//doubleArr[2] = 9.9;
Array array = conn.createArrayOf("DOUBLE", doubleArr);
stmt.setArray(4, array);
+
+ // create character array
+ String[] charArr = new String[2];
+ charArr[0] = "a";
+ charArr[1] = "b";
+ array = conn.createArrayOf("CHAR", charArr);
+ stmt.setArray(5, array);
stmt.execute();
conn.commit();