You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by td...@apache.org on 2016/12/23 01:28:09 UTC
[35/42] phoenix git commit: PHOENIX-3442 Support null when columns
have default values for immutable tables with encoding scheme
COLUMNS_STORED_IN_SINGLE_CELL
PHOENIX-3442 Support null when columns have default values for immutable tables with encoding scheme COLUMNS_STORED_IN_SINGLE_CELL
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/01ef5d5b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/01ef5d5b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/01ef5d5b
Branch: refs/heads/encodecolumns2
Commit: 01ef5d5b4a298e203cbc43487cc421bd920fbbac
Parents: aa7450f
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Wed Nov 23 19:11:25 2016 -0800
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Thu Dec 22 13:00:44 2016 -0800
----------------------------------------------------------------------
.../phoenix/end2end/DefaultColumnValueIT.java | 2 +-
.../expression/ArrayColumnExpression.java | 3 +-
.../expression/ArrayConstructorExpression.java | 67 ++---
.../expression/util/regex/JONIPattern.java | 2 +-
.../org/apache/phoenix/schema/ColumnRef.java | 8 +-
.../org/apache/phoenix/schema/PTableImpl.java | 29 ++-
.../phoenix/schema/types/PArrayDataType.java | 244 +++++++++++--------
.../ArrayConstructorExpressionTest.java | 106 +++++++-
8 files changed, 289 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultColumnValueIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultColumnValueIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultColumnValueIT.java
index 8302604..7c04d01 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultColumnValueIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultColumnValueIT.java
@@ -258,7 +258,7 @@ public class DefaultColumnValueIT extends ParallelStatsDisabledIT {
assertFalse(rs.next());
}
- @Ignore //FIXME: PHOENIX-3442
+ @Test
public void testDefaultImmutableRows() throws Exception {
String table = generateUniqueName();
String ddl = "CREATE TABLE IF NOT EXISTS " + table + " (" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
index f09fb62..747d7e6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
@@ -74,8 +74,7 @@ public class ArrayColumnExpression extends KeyValueColumnExpression {
// Given a ptr to the entire array, set ptr to point to a particular element within that array
// given the type of an array element (see comments in PDataTypeForArray)
- PArrayDataType.positionAtArrayElement(ptr, positionInArray, PVarbinary.INSTANCE, null);
- return true;
+ return PArrayDataType.positionAtArrayElement(ptr, positionInArray, PVarbinary.INSTANCE, null);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
index 783e962..2a6b484 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.expression.visitor.ExpressionVisitor;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PArrayDataType;
+import org.apache.phoenix.schema.types.PArrayDataType.PArrayDataTypeBytesArrayBuilder;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.TrustedByteArrayOutputStream;
@@ -31,7 +32,6 @@ import org.apache.phoenix.util.TrustedByteArrayOutputStream;
public class ArrayConstructorExpression extends BaseCompoundExpression {
private PDataType baseType;
private int position = -1;
- private int nNulls = 0;
private Object[] elements;
private final ImmutableBytesWritable valuePtr = new ImmutableBytesWritable();
private int estimatedSize = 0;
@@ -39,20 +39,25 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
// and serialize into byte stream
private int[] offsetPos;
private boolean rowKeyOrderOptimizable;
+ private byte serializationVersion;
public ArrayConstructorExpression() {
}
public ArrayConstructorExpression(List<Expression> children, PDataType baseType, boolean rowKeyOrderOptimizable) {
+ this(children, baseType, rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION);
+ }
+
+ public ArrayConstructorExpression(List<Expression> children, PDataType baseType, boolean rowKeyOrderOptimizable, byte serializationVersion) {
super(children);
- init(baseType, rowKeyOrderOptimizable);
+ init(baseType, rowKeyOrderOptimizable, serializationVersion);
}
public ArrayConstructorExpression clone(List<Expression> children) {
- return new ArrayConstructorExpression(children, this.baseType, this.rowKeyOrderOptimizable);
+ return new ArrayConstructorExpression(children, this.baseType, this.rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION);
}
- private void init(PDataType baseType, boolean rowKeyOrderOptimizable) {
+ private void init(PDataType baseType, boolean rowKeyOrderOptimizable, byte serializationVersion) {
this.baseType = baseType;
this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
elements = new Object[getChildren().size()];
@@ -61,6 +66,7 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
if (!this.baseType.isFixedWidth()) {
offsetPos = new int[children.size()];
}
+ this.serializationVersion = serializationVersion;
}
@Override
@@ -72,7 +78,6 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
public void reset() {
super.reset();
position = 0;
- nNulls = 0;
Arrays.fill(elements, null);
valuePtr.set(ByteUtil.EMPTY_BYTE_ARRAY);
}
@@ -85,9 +90,9 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
}
TrustedByteArrayOutputStream byteStream = new TrustedByteArrayOutputStream(estimatedSize);
DataOutputStream oStream = new DataOutputStream(byteStream);
+ PArrayDataTypeBytesArrayBuilder builder =
+ new PArrayDataTypeBytesArrayBuilder(byteStream, oStream, children.size(), baseType, getSortOrder(), rowKeyOrderOptimizable, serializationVersion);
try {
- int noOfElements = children.size();
- nNulls = 0;
for (int i = position >= 0 ? position : 0; i < elements.length; i++) {
Expression child = children.get(i);
if (!child.evaluate(tuple, ptr)) {
@@ -95,48 +100,19 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
if (position >= 0) position = i;
return false;
}
- } else {
- // track the offset position here from the size of the byteStream
- if (!baseType.isFixedWidth()) {
- // Any variable length array would follow the below order
- // Every element would be seperated by a seperator byte '0'
- // Null elements are counted and once a first non null element appears we
- // write the count of the nulls prefixed with a seperator byte
- // Trailing nulls are not taken into account
- // The last non null element is followed by two seperator bytes
- // For eg
- // a, b, null, null, c, null would be
- // 65 0 66 0 0 2 67 0 0 0
- // a null null null b c null d would be
- // 65 0 0 3 66 0 67 0 0 1 68 0 0 0
- if (ptr.getLength() == 0) {
- offsetPos[i] = byteStream.size();
- nNulls++;
- } else {
- PArrayDataType.serializeNulls(oStream, nNulls);
- offsetPos[i] = byteStream.size();
- oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength());
- oStream.write(PArrayDataType.getSeparatorByte(rowKeyOrderOptimizable, getSortOrder()));
- nNulls = 0;
- }
- } else { // No nulls for fixed length
- oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength());
+ else {
+ // its possible for the expression to evaluate to null if the serialization format is immutable and the data type is variable length
+ builder.appendMissingElement();
}
+ } else {
+ builder.appendElem(ptr.get(), ptr.getOffset(), ptr.getLength());
}
}
if (position >= 0) position = elements.length;
- if (!baseType.isFixedWidth()) {
- // Double seperator byte to show end of the non null array
- PArrayDataType.writeEndSeperatorForVarLengthArray(oStream, getSortOrder(), rowKeyOrderOptimizable);
- noOfElements = PArrayDataType.serailizeOffsetArrayIntoStream(oStream, byteStream, noOfElements,
- offsetPos[offsetPos.length - 1], offsetPos);
- PArrayDataType.serializeHeaderInfoIntoStream(oStream, noOfElements);
- }
- ptr.set(byteStream.getBuffer(), 0, byteStream.size());
+ byte[] bytes = builder.getBytesAndClose();
+ ptr.set(bytes, 0, bytes.length);
valuePtr.set(ptr.get(), ptr.getOffset(), ptr.getLength());
return true;
- } catch (IOException e) {
- throw new RuntimeException("Exception while serializing the byte array");
} finally {
try {
byteStream.close();
@@ -157,7 +133,8 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
rowKeyOrderOptimizable = true;
baseTypeOrdinal = -(baseTypeOrdinal+1);
}
- init(PDataType.values()[baseTypeOrdinal], rowKeyOrderOptimizable);
+ byte serializationVersion = input.readByte();
+ init(PDataType.values()[baseTypeOrdinal], rowKeyOrderOptimizable, serializationVersion);
}
@Override
@@ -168,6 +145,7 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
} else {
WritableUtils.writeVInt(output, baseType.ordinal());
}
+ output.write(serializationVersion);
}
@Override
@@ -196,4 +174,5 @@ public class ArrayConstructorExpression extends BaseCompoundExpression {
buf.append(children.get(children.size()-1) + "]");
return buf.toString();
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java
index af5bc2b..522a4e7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java
@@ -192,7 +192,7 @@ public class JONIPattern extends AbstractBasePattern implements AbstractBaseSpli
break;
}
}
- byte[] bytes = builder.getBytesAndClose(SortOrder.ASC);
+ byte[] bytes = builder.getBytesAndClose();
if (bytes == null) return false;
outPtr.set(bytes);
return true;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
index 256575c..d757322 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
@@ -128,7 +128,8 @@ public class ColumnRef {
return new ProjectedColumnExpression(column, table, displayName);
}
- Expression expression = new KeyValueColumnExpression(column, displayName, usesEncodedColumnNames(table));
+ Expression expression = table.getStorageScheme() == StorageScheme.ONE_CELL_PER_COLUMN_FAMILY ?
+ new ArrayColumnExpression(column, displayName, EncodedColumnsUtil.usesEncodedColumnNames(table)) : new KeyValueColumnExpression(column, displayName, usesEncodedColumnNames(table));
if (column.getExpressionStr() != null) {
String url = PhoenixRuntime.JDBC_PROTOCOL
@@ -146,10 +147,7 @@ public class ColumnRef {
}
}
- if (table.getStorageScheme() == StorageScheme.ONE_CELL_PER_COLUMN_FAMILY) {
- return new ArrayColumnExpression(column, displayName, EncodedColumnsUtil.usesEncodedColumnNames(table.getEncodingScheme()));
- }
- return new KeyValueColumnExpression(column, displayName, EncodedColumnsUtil.usesEncodedColumnNames(table.getEncodingScheme()));
+ return expression;
}
public ColumnRef cloneAtTimestamp(long timestamp) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 8522c13..7e6f35b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
@@ -49,6 +50,7 @@ import org.apache.phoenix.compile.StatementContext;
import org.apache.phoenix.coprocessor.generated.PTableProtos;
import org.apache.phoenix.exception.DataExceedsCapacityException;
import org.apache.phoenix.expression.ArrayConstructorExpression;
+import org.apache.phoenix.expression.DelegateExpression;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.LiteralExpression;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
@@ -61,7 +63,8 @@ import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.protobuf.ProtobufUtil;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
-import org.apache.phoenix.schema.tuple.BaseTuple;
+import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PArrayDataType;
import org.apache.phoenix.schema.types.PBinary;
import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PDataType;
@@ -897,21 +900,27 @@ public class PTableImpl implements PTable {
for (PColumn column : columns) {
maxEncodedColumnQualifier = Math.max(maxEncodedColumnQualifier, column.getEncodedColumnQualifier());
}
- byte[][] colValues = new byte[maxEncodedColumnQualifier+1][];
+ Expression[] colValues = new Expression[maxEncodedColumnQualifier+1];
+ Arrays.fill(colValues, new DelegateExpression(LiteralExpression.newConstant(null)) {
+ @Override
+ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
+ return false;
+ }
+ });
+ // 0 is a reserved position, set it to a non-null value so that we can represent absence of a value using a negative offset
+ colValues[0]=LiteralExpression.newConstant(QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
for (PColumn column : columns) {
- colValues[column.getEncodedColumnQualifier()] = columnToValueMap.get(column);
+ if (columnToValueMap.containsKey(column)) {
+ colValues[column.getEncodedColumnQualifier()] = new LiteralExpression(columnToValueMap.get(column));
+ }
}
- List<Expression> children = Lists.newArrayListWithExpectedSize(columns.size());
- // create an expression list with all the columns
- for (int i=0; i<colValues.length; ++i) {
- children.add(new LiteralExpression(colValues[i]==null ? ByteUtil.EMPTY_BYTE_ARRAY : colValues[i] ));
- }
+ List<Expression> children = Arrays.asList(colValues);
// we use ArrayConstructorExpression to serialize multiple columns into a single byte[]
// construct the ArrayConstructorExpression with a variable length data type since columns can be of fixed or variable length
- ArrayConstructorExpression arrayExpression = new ArrayConstructorExpression(children, PVarbinary.INSTANCE, rowKeyOrderOptimizable);
+ ArrayConstructorExpression arrayExpression = new ArrayConstructorExpression(children, PVarbinary.INSTANCE, rowKeyOrderOptimizable, PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- arrayExpression.evaluate(new BaseTuple() {}, ptr);
+ arrayExpression.evaluate(null, ptr);
ImmutableBytesPtr colFamilyPtr = new ImmutableBytesPtr(columnFamily);
addQuietly(put, kvBuilder, kvBuilder.buildPut(keyPtr,
colFamilyPtr, QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES_PTR, ts, ptr));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
index 1d2cfb2..fede7d8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.sql.Types;
import java.text.Format;
+import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;
@@ -74,8 +75,11 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
this, actualModifer, desiredModifier, true);
}
- public static final byte ARRAY_SERIALIZATION_VERSION = 1;
-
+ // array serialization format where bytes can be used as part of the row key
+ public static final byte SORTABLE_SERIALIZATION_VERSION = 1;
+ // array serialization format where bytes are immutable (does not support prepend/append or sorting)
+ public static final byte IMMUTABLE_SERIALIZATION_VERSION = 2;
+
protected PArrayDataType(String sqlTypeName, int sqlType, Class clazz, PDataCodec codec, int ordinal) {
super(sqlTypeName, sqlType, clazz, codec, ordinal);
}
@@ -186,9 +190,17 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
oStream.write(sepByte);
}
- public static boolean useShortForOffsetArray(int maxOffset) {
- // If the max offset is less than Short.MAX_VALUE then offset array can use short
- if (maxOffset <= (2 * Short.MAX_VALUE)) { return true; }
+ // this method is only for append/prepend/concat operations which are only supported for the SORTABLE_SERIALIZATION_VERSION
+ public static boolean useShortForOffsetArray(int maxoffset) {
+ return useShortForOffsetArray(maxoffset, SORTABLE_SERIALIZATION_VERSION);
+ }
+
+ public static boolean useShortForOffsetArray(int maxoffset, byte serializationVersion) {
+ if (serializationVersion == IMMUTABLE_SERIALIZATION_VERSION) {
+ return (maxoffset <= Short.MAX_VALUE && maxoffset >= Short.MIN_VALUE );
+ }
+ // If the max offset is less than Short.MAX_VALUE then offset array can use short
+ else if (maxoffset <= (2 * Short.MAX_VALUE)) { return true; }
// else offset array can use Int
return false;
}
@@ -350,15 +362,15 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// Given a ptr to the entire array, set ptr to point to a particular element within that array
// given the type of an array element (see comments in PDataTypeForArray)
- positionAtArrayElement(ptr, index - 1, pDataType, maxLen);
- return true;
+ return positionAtArrayElement(ptr, index - 1, pDataType, maxLen);
}
- public static void positionAtArrayElement(ImmutableBytesWritable ptr, int arrayIndex, PDataType baseDataType,
+ public static boolean positionAtArrayElement(ImmutableBytesWritable ptr, int arrayIndex, PDataType baseDataType,
Integer byteSize) {
byte[] bytes = ptr.get();
int initPos = ptr.getOffset();
if (!baseDataType.isFixedWidth()) {
+ byte serializationVersion = bytes[ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_BYTE];
int noOfElements = Bytes.toInt(bytes,
(ptr.getOffset() + ptr.getLength() - (Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT)), Bytes.SIZEOF_INT);
boolean useShort = true;
@@ -368,13 +380,14 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
if (arrayIndex >= noOfElements) {
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
- return;
+ return false;
}
int indexOffset = Bytes.toInt(bytes,
(ptr.getOffset() + ptr.getLength() - (Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT))) + ptr.getOffset();
if (arrayIndex >= noOfElements) {
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
+ return false;
} else {
// Skip those many offsets as given in the arrayIndex
// If suppose there are 5 elements in the array and the arrayIndex = 3
@@ -383,14 +396,20 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// offset of 5th element.
// Subtracting the offset of 5th element and 4th element will give the length of 4th element
// So we could just skip reading the other elements.
- int currOffset = getOffset(bytes, arrayIndex, useShort, indexOffset);
+ int currOffset = getSerializedOffset(bytes, arrayIndex, useShort, indexOffset, serializationVersion);
+ if (currOffset<0) {
+ ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
+ return false;
+ }
int elementLength = 0;
if (arrayIndex == (noOfElements - 1)) {
+ int separatorBytes = serializationVersion == SORTABLE_SERIALIZATION_VERSION ? 3 : 0;
elementLength = (bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : indexOffset
- - (currOffset + initPos) - 3;
+ - (currOffset + initPos) - separatorBytes;
} else {
+ int separatorByte = serializationVersion == SORTABLE_SERIALIZATION_VERSION ? 1 : 0;
elementLength = (bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : getOffset(bytes,
- arrayIndex + 1, useShort, indexOffset) - currOffset - 1;
+ arrayIndex + 1, useShort, indexOffset, serializationVersion) - currOffset - separatorByte;
}
ptr.set(bytes, currOffset + initPos, elementLength);
}
@@ -403,12 +422,14 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
ptr.set(bytes, ptr.getOffset() + offset, elemByteSize);
}
}
+ return true;
}
public static void positionAtArrayElement(ImmutableBytesWritable ptr, int arrayIndex, PDataType baseDataType,
Integer byteSize, int offset, int length, int noOfElements, boolean first) {
byte[] bytes = ptr.get();
if (!baseDataType.isFixedWidth()) {
+ byte serializationVersion = bytes[ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_BYTE];
int indexOffset = Bytes.toInt(bytes, (offset + length - (Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT)))
+ offset;
boolean useShort = true;
@@ -430,14 +451,14 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// offset of 5th element.
// Subtracting the offset of 5th element and 4th element will give the length of 4th element
// So we could just skip reading the other elements.
- int currOffset = getOffset(bytes, arrayIndex, useShort, indexOffset);
+ int currOffset = getOffset(bytes, arrayIndex, useShort, indexOffset, serializationVersion);
int elementLength = 0;
if (arrayIndex == (noOfElements - 1)) {
elementLength = (bytes[currOffset + offset] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + offset] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : indexOffset
- (currOffset + offset) - 3;
} else {
elementLength = (bytes[currOffset + offset] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + offset] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : getOffset(bytes,
- arrayIndex + 1, useShort, indexOffset) - currOffset - 1;
+ arrayIndex + 1, useShort, indexOffset, serializationVersion) - currOffset - 1;
}
ptr.set(bytes, currOffset + offset, elementLength);
}
@@ -452,16 +473,20 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
}
- private static int getOffset(byte[] bytes, int arrayIndex, boolean useShort, int indexOffset) {
- int offset;
+ private static int getOffset(byte[] bytes, int arrayIndex, boolean useShort, int indexOffset, byte serializationVersion) {
+ return Math.abs(getSerializedOffset(bytes, arrayIndex, useShort, indexOffset, serializationVersion));
+ }
+
+ private static int getSerializedOffset(byte[] bytes, int arrayIndex, boolean useShort, int indexOffset, byte serializationVersion) {
+ int offset;
if (useShort) {
offset = indexOffset + (Bytes.SIZEOF_SHORT * arrayIndex);
- return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT) + Short.MAX_VALUE;
+ return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT) + (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION ? 0 : Short.MAX_VALUE);
} else {
offset = indexOffset + (Bytes.SIZEOF_INT * arrayIndex);
return Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
}
- }
+ }
private static int getOffset(ByteBuffer indexBuffer, int arrayIndex, boolean useShort, int indexOffset) {
int offset;
@@ -484,58 +509,18 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
/**
- * creates array bytes
+ * creates array bytes using the SORTABLE_SERIALIZATION_VERSION format
* @param rowKeyOrderOptimizable TODO
*/
private byte[] createArrayBytes(TrustedByteArrayOutputStream byteStream, DataOutputStream oStream,
PhoenixArray array, int noOfElements, PDataType baseType, SortOrder sortOrder, boolean rowKeyOrderOptimizable) {
- try {
- if (!baseType.isFixedWidth()) {
- int[] offsetPos = new int[noOfElements];
- int nulls = 0;
- for (int i = 0; i < noOfElements; i++) {
- byte[] bytes = array.toBytes(i);
- if (bytes.length == 0) {
- offsetPos[i] = byteStream.size();
- nulls++;
- } else {
- nulls = serializeNulls(oStream, nulls);
- offsetPos[i] = byteStream.size();
- if (sortOrder == SortOrder.DESC) {
- SortOrder.invert(bytes, 0, bytes, 0, bytes.length);
- }
- oStream.write(bytes, 0, bytes.length);
- oStream.write(getSeparatorByte(rowKeyOrderOptimizable, sortOrder));
- }
- }
- // Double seperator byte to show end of the non null array
- writeEndSeperatorForVarLengthArray(oStream, sortOrder, rowKeyOrderOptimizable);
- noOfElements = PArrayDataType.serailizeOffsetArrayIntoStream(oStream, byteStream, noOfElements,
- offsetPos[offsetPos.length - 1], offsetPos);
- serializeHeaderInfoIntoStream(oStream, noOfElements);
- } else {
- for (int i = 0; i < noOfElements; i++) {
- byte[] bytes = array.toBytes(i);
- int length = bytes.length;
- if (sortOrder == SortOrder.DESC) {
- SortOrder.invert(bytes, 0, bytes, 0, bytes.length);
- }
- oStream.write(bytes, 0, length);
- }
- }
- ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- ptr.set(byteStream.getBuffer(), 0, byteStream.size());
- return ByteUtil.copyKeyBytesIfNecessary(ptr);
- } catch (IOException e) {
- try {
- byteStream.close();
- oStream.close();
- } catch (IOException ioe) {
-
- }
+ PArrayDataTypeBytesArrayBuilder builder =
+ new PArrayDataTypeBytesArrayBuilder(byteStream, oStream, noOfElements, baseType, sortOrder, rowKeyOrderOptimizable);
+ for (int i = 0; i < noOfElements; i++) {
+ byte[] bytes = array.toBytes(i);
+ builder.appendElem(bytes);
}
- // This should not happen
- return null;
+ return builder.getBytesAndClose();
}
public static boolean appendItemToArray(ImmutableBytesWritable ptr, int length, int offset, byte[] arrayBytes,
@@ -557,7 +542,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
byte[] newArray;
if (!baseType.isFixedWidth()) {
-
+ byte serializationVersion = arrayBytes[offset + length - Bytes.SIZEOF_BYTE];
int offsetArrayPosition = Bytes.toInt(arrayBytes, offset + length - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT
- Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT);
int offsetArrayLength = length - offsetArrayPosition - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT
@@ -612,7 +597,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
int off = newOffsetArrayPosition;
for (int arrayIndex = 0; arrayIndex < Math.abs(arrayLength) - 1; arrayIndex++) {
Bytes.putInt(newArray, off,
- getOffset(arrayBytes, arrayIndex, true, offsetArrayPosition + offset));
+ getOffset(arrayBytes, arrayIndex, true, offsetArrayPosition + offset, serializationVersion));
off += Bytes.SIZEOF_INT;
}
@@ -659,6 +644,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
byte[] newArray;
if (!baseType.isFixedWidth()) {
+ byte serializationVersion = arrayBytes[offset + length - Bytes.SIZEOF_BYTE];
int offsetArrayPosition = Bytes.toInt(arrayBytes, offset + length - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT
- Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT);
int offsetArrayLength = length - offsetArrayPosition - Bytes.SIZEOF_INT - Bytes.SIZEOF_INT
@@ -668,7 +654,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// checks whether offset array consists of shorts or integers
boolean useInt = offsetArrayLength / arrayLength == Bytes.SIZEOF_INT;
boolean convertToInt = false;
- int endElementPosition = getOffset(arrayBytes, arrayLength - 1, !useInt, offsetArrayPosition + offset)
+ int endElementPosition = getOffset(arrayBytes, arrayLength - 1, !useInt, offsetArrayPosition + offset, serializationVersion)
+ elementLength + Bytes.SIZEOF_BYTE;
int newOffsetArrayPosition;
int lengthIncrease;
@@ -679,7 +665,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
int nulls = 1;
// counts the number of nulls which are already at the beginning of the array
for (int index = 0; index < arrayLength; index++) {
- int currOffset = getOffset(arrayBytes, index, !useInt, offsetArrayPosition + offset);
+ int currOffset = getOffset(arrayBytes, index, !useInt, offsetArrayPosition + offset, serializationVersion);
if (arrayBytes[offset + currOffset] == QueryConstants.SEPARATOR_BYTE) {
nulls++;
} else {
@@ -709,7 +695,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// ex: initial array - 0 45(inverted) 65 0 66 0 0 0 after prepending null - 0 46(inverted) 65 0 66 0 0 0
lengthIncrease = nRemainingNulls == 1 ? (nMultiplesOver255 == 0 ? 2 * Bytes.SIZEOF_BYTE
: Bytes.SIZEOF_BYTE) : 0;
- endElementPosition = getOffset(arrayBytes, arrayLength - 1, !useInt, offsetArrayPosition + offset)
+ endElementPosition = getOffset(arrayBytes, arrayLength - 1, !useInt, offsetArrayPosition + offset, serializationVersion)
+ lengthIncrease;
if (!useInt) {
if (PArrayDataType.useShortForOffsetArray(endElementPosition)) {
@@ -785,8 +771,9 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
currentPosition += offsetArrayElementSize;
boolean nullsAtBeginning = true;
+ byte serializationVersion = arrayBytes[offset + length - Bytes.SIZEOF_BYTE];
for (int arrayIndex = 0; arrayIndex < arrayLength - 1; arrayIndex++) {
- int oldOffset = getOffset(arrayBytes, arrayIndex, useShortPrevious, offsetArrayPosition + offset);
+ int oldOffset = getOffset(arrayBytes, arrayIndex, useShortPrevious, offsetArrayPosition + offset, serializationVersion);
if (arrayBytes[offset + oldOffset] == QueryConstants.SEPARATOR_BYTE && nullsAtBeginning) {
if (useShortNew) {
Bytes.putShort(newArray, currentPosition, (short)(oldOffset - Short.MAX_VALUE));
@@ -820,6 +807,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
byte[] newArray;
if (!baseType.isFixedWidth()) {
+ byte serializationVersion1 = array1Bytes[array1BytesOffset + array1BytesLength - Bytes.SIZEOF_BYTE];
int offsetArrayPositionArray1 = Bytes.toInt(array1Bytes, array1BytesOffset + array1BytesLength
- Bytes.SIZEOF_INT - Bytes.SIZEOF_INT - Bytes.SIZEOF_BYTE, Bytes.SIZEOF_INT);
int offsetArrayPositionArray2 = Bytes.toInt(array2Bytes, array2BytesOffset + array2BytesLength
@@ -837,7 +825,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
boolean useIntNewArray = false;
// count nulls at the end of array 1
for (int index = actualLengthOfArray1 - 1; index > -1; index--) {
- int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1);
+ int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset + offsetArrayPositionArray1, serializationVersion1);
if (array1Bytes[array1BytesOffset + offset] == QueryConstants.SEPARATOR_BYTE || array1Bytes[array1BytesOffset + offset] == QueryConstants.DESC_SEPARATOR_BYTE) {
nullsAtTheEndOfArray1++;
} else {
@@ -847,8 +835,9 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// count nulls at the beginning of the array 2
int array2FirstNonNullElementOffset = 0;
int array2FirstNonNullIndex = 0;
+ byte serializationVersion2 = array2Bytes[array2BytesOffset + array2BytesLength - Bytes.SIZEOF_BYTE];
for (int index = 0; index < actualLengthOfArray2; index++) {
- int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2);
+ int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset + offsetArrayPositionArray2, serializationVersion2);
if (array2Bytes[array2BytesOffset + offset] == QueryConstants.SEPARATOR_BYTE) {
nullsAtTheBeginningOfArray2++;
} else {
@@ -870,7 +859,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
int newOffsetArrayPosition = offsetArrayPositionArray1 + offsetArrayPositionArray2 + lengthIncreaseForNulls
- 2 * Bytes.SIZEOF_BYTE;
int endElementPositionOfArray2 = getOffset(array2Bytes, actualLengthOfArray2 - 1, !useIntArray2,
- array2BytesOffset + offsetArrayPositionArray2);
+ array2BytesOffset + offsetArrayPositionArray2, serializationVersion2);
int newEndElementPosition = lengthIncreaseForNulls + endElementPositionOfArray2 + offsetArrayPositionArray1
- 2 * Bytes.SIZEOF_BYTE;
// Creates a byte array to store the concatenated array
@@ -902,14 +891,14 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// offsets for the elements from array 1. Simply copied.
for (int index = 0; index < actualLengthOfArray1; index++) {
int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset
- + offsetArrayPositionArray1);
+ + offsetArrayPositionArray1, serializationVersion1);
Bytes.putInt(newArray, currentPosition, offset);
currentPosition += Bytes.SIZEOF_INT;
}
// offsets for nulls in the middle
for (int index = 0; index < array2FirstNonNullIndex; index++) {
int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset
- + offsetArrayPositionArray2);
+ + offsetArrayPositionArray2, serializationVersion2);
Bytes.putInt(newArray, currentPosition, offset + array2StartingPosition);
currentPosition += Bytes.SIZEOF_INT;
}
@@ -918,7 +907,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
+ (bytesForNullsAfter == 0 ? 0 : Bytes.SIZEOF_BYTE);
for (int index = array2FirstNonNullIndex; index < actualLengthOfArray2; index++) {
int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset
- + offsetArrayPositionArray2);
+ + offsetArrayPositionArray2, serializationVersion2);
Bytes.putInt(newArray, currentPosition, offset - array2FirstNonNullElementOffset
+ part2NonNullStartingPosition);
currentPosition += Bytes.SIZEOF_INT;
@@ -927,14 +916,14 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// offsets for the elements from array 1. Simply copied.
for (int index = 0; index < actualLengthOfArray1; index++) {
int offset = getOffset(array1Bytes, index, !useIntArray1, array1BytesOffset
- + offsetArrayPositionArray1);
+ + offsetArrayPositionArray1, serializationVersion1);
Bytes.putShort(newArray, currentPosition, (short)(offset - Short.MAX_VALUE));
currentPosition += Bytes.SIZEOF_SHORT;
}
// offsets for nulls in the middle
for (int index = 0; index < array2FirstNonNullIndex; index++) {
int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset
- + offsetArrayPositionArray2);
+ + offsetArrayPositionArray2, serializationVersion2);
Bytes.putShort(newArray, currentPosition,
(short)(offset + array2StartingPosition - Short.MAX_VALUE));
currentPosition += Bytes.SIZEOF_SHORT;
@@ -944,7 +933,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
+ (bytesForNullsAfter == 0 ? 0 : Bytes.SIZEOF_BYTE);
for (int index = array2FirstNonNullIndex; index < actualLengthOfArray2; index++) {
int offset = getOffset(array2Bytes, index, !useIntArray2, array2BytesOffset
- + offsetArrayPositionArray2);
+ + offsetArrayPositionArray2, serializationVersion2);
Bytes.putShort(newArray, currentPosition, (short)(offset - array2FirstNonNullElementOffset
+ part2NonNullStartingPosition - Short.MAX_VALUE));
currentPosition += Bytes.SIZEOF_SHORT;
@@ -1013,13 +1002,13 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
ptr.set(PVarcharArray.INSTANCE.toBytes(phoenixArray, PVarchar.INSTANCE, sortOrder));
return true;
}
-
- public static int serailizeOffsetArrayIntoStream(DataOutputStream oStream, TrustedByteArrayOutputStream byteStream,
- int noOfElements, int maxOffset, int[] offsetPos) throws IOException {
+
+ public static int serializeOffsetArrayIntoStream(DataOutputStream oStream, TrustedByteArrayOutputStream byteStream,
+ int noOfElements, int maxOffset, int[] offsetPos, byte serializationVersion) throws IOException {
int offsetPosition = (byteStream.size());
byte[] offsetArr = null;
boolean useInt = true;
- if (PArrayDataType.useShortForOffsetArray(maxOffset)) {
+ if (PArrayDataType.useShortForOffsetArray(maxOffset, serializationVersion)) {
offsetArr = new byte[PArrayDataType.initOffsetArray(noOfElements, Bytes.SIZEOF_SHORT)];
useInt = false;
} else {
@@ -1034,7 +1023,8 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
} else {
for (int pos : offsetPos) {
- Bytes.putShort(offsetArr, off, (short)(pos - Short.MAX_VALUE));
+ short val = serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION ? (short)pos : (short)(pos - Short.MAX_VALUE);
+ Bytes.putShort(offsetArr, off, val);
off += Bytes.SIZEOF_SHORT;
}
}
@@ -1043,18 +1033,11 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
return noOfElements;
}
- public static void serializeHeaderInfoIntoBuffer(ByteBuffer buffer, int noOfElements) {
- // No of elements
- buffer.putInt(noOfElements);
- // Version of the array
- buffer.put(ARRAY_SERIALIZATION_VERSION);
- }
-
- public static void serializeHeaderInfoIntoStream(DataOutputStream oStream, int noOfElements) throws IOException {
+ public static void serializeHeaderInfoIntoStream(DataOutputStream oStream, int noOfElements, byte serializationVersion) throws IOException {
// No of elements
oStream.writeInt(noOfElements);
// Version of the array
- oStream.write(ARRAY_SERIALIZATION_VERSION);
+ oStream.write(serializationVersion);
}
public static int initOffsetArray(int noOfElements, int baseSize) {
@@ -1229,8 +1212,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
return buf.toString();
}
- // FIXME: remove this duplicate code
- static public class PArrayDataTypeBytesArrayBuilder<T> {
+ static public class PArrayDataTypeBytesArrayBuilder {
static private final int BYTE_ARRAY_DEFAULT_SIZE = 128;
private PDataType baseType;
@@ -1239,14 +1221,38 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
private TrustedByteArrayOutputStream byteStream;
private DataOutputStream oStream;
private int nulls;
+ private byte serializationVersion;
+ private boolean rowKeyOrderOptimizable;
public PArrayDataTypeBytesArrayBuilder(PDataType baseType, SortOrder sortOrder) {
+ this(new TrustedByteArrayOutputStream(BYTE_ARRAY_DEFAULT_SIZE), new LinkedList<Integer>(), baseType, sortOrder, true);
+ }
+
+ public PArrayDataTypeBytesArrayBuilder(TrustedByteArrayOutputStream byteStream, DataOutputStream oStream,
+ int numElements, PDataType baseType, SortOrder sortOrder, boolean rowKeyOrderOptimizable, byte serializationVersion) {
+ this(byteStream, oStream, new ArrayList<Integer>(numElements), baseType, sortOrder, rowKeyOrderOptimizable, serializationVersion);
+ }
+
+ public PArrayDataTypeBytesArrayBuilder(TrustedByteArrayOutputStream byteStream, DataOutputStream oStream,
+ int numElements, PDataType baseType, SortOrder sortOrder, boolean rowKeyOrderOptimizable) {
+ this(byteStream, oStream, new ArrayList<Integer>(numElements), baseType, sortOrder, rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION);
+ }
+
+ public PArrayDataTypeBytesArrayBuilder(TrustedByteArrayOutputStream byteStream,
+ List<Integer> offsetPos, PDataType baseType, SortOrder sortOrder, boolean rowKeyOrderOptimizable) {
+ this(byteStream, new DataOutputStream(byteStream), offsetPos, baseType, sortOrder, rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION);
+ }
+
+ public PArrayDataTypeBytesArrayBuilder(TrustedByteArrayOutputStream byteStream, DataOutputStream oStream,
+ List<Integer> offsetPos, PDataType baseType, SortOrder sortOrder, boolean rowKeyOrderOptimizable, byte serializationVersion) {
this.baseType = baseType;
this.sortOrder = sortOrder;
- offsetPos = new LinkedList<Integer>();
- byteStream = new TrustedByteArrayOutputStream(BYTE_ARRAY_DEFAULT_SIZE);
- oStream = new DataOutputStream(byteStream);
- nulls = 0;
+ this.offsetPos = offsetPos;
+ this.byteStream = byteStream;
+ this.oStream = oStream;
+ this.nulls = 0;
+ this.serializationVersion = serializationVersion;
+ this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
}
private void close() {
@@ -1257,6 +1263,14 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
oStream = null;
} catch (IOException ioe) {}
}
+
+ // used to represent the absence of a value
+ public void appendMissingElement() {
+ if (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION && !baseType.isFixedWidth()) {
+ offsetPos.add(-byteStream.size());
+ nulls++;
+ }
+ }
public boolean appendElem(byte[] bytes) {
return appendElem(bytes, 0, bytes.length);
@@ -1265,7 +1279,19 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
public boolean appendElem(byte[] bytes, int offset, int len) {
if (oStream == null || byteStream == null) return false;
try {
+ // track the offset position here from the size of the byteStream
if (!baseType.isFixedWidth()) {
+ // Any variable length array would follow the below order
+ // Every element would be seperated by a seperator byte '0'
+ // Null elements are counted and once a first non null element appears we
+ // write the count of the nulls prefixed with a seperator byte
+ // Trailing nulls are not taken into account
+ // The last non null element is followed by two seperator bytes
+ // For eg
+ // a, b, null, null, c, null would be
+ // 65 0 66 0 0 2 67 0 0 0
+ // a null null null b c null d would be
+ // 65 0 0 3 66 0 67 0 0 1 68 0 0 0
if (len == 0) {
offsetPos.add(byteStream.size());
nulls++;
@@ -1277,9 +1303,12 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
offset = 0;
}
oStream.write(bytes, offset, len);
- oStream.write(getSeparatorByte(true, sortOrder));
+ if (serializationVersion == SORTABLE_SERIALIZATION_VERSION) {
+ oStream.write(getSeparatorByte(rowKeyOrderOptimizable, sortOrder));
+ }
}
} else {
+ // No nulls for fixed length
if (sortOrder == SortOrder.DESC) {
SortOrder.invert(bytes, offset, bytes, offset, len);
offset = 0;
@@ -1291,7 +1320,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
return false;
}
- public byte[] getBytesAndClose(SortOrder sortOrder) {
+ public byte[] getBytesAndClose() {
try {
if (!baseType.isFixedWidth()) {
int noOfElements = offsetPos.size();
@@ -1301,10 +1330,13 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
offsetPosArray[index] = i;
++index;
}
- PArrayDataType.writeEndSeperatorForVarLengthArray(oStream, sortOrder);
- noOfElements = PArrayDataType.serailizeOffsetArrayIntoStream(oStream, byteStream, noOfElements,
- offsetPosArray[offsetPosArray.length - 1], offsetPosArray);
- serializeHeaderInfoIntoStream(oStream, noOfElements);
+ if (serializationVersion == SORTABLE_SERIALIZATION_VERSION) {
+ // Double seperator byte to show end of the non null array
+ writeEndSeperatorForVarLengthArray(oStream, sortOrder, rowKeyOrderOptimizable);
+ }
+ noOfElements = PArrayDataType.serializeOffsetArrayIntoStream(oStream, byteStream, noOfElements,
+ offsetPosArray[offsetPosArray.length - 1], offsetPosArray, serializationVersion);
+ serializeHeaderInfoIntoStream(oStream, noOfElements, serializationVersion);
}
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
ptr.set(byteStream.getBuffer(), 0, byteStream.size());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/01ef5d5b/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java
index e99a71c..a78e87e 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java
@@ -18,11 +18,17 @@
package org.apache.phoenix.expression;
import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.util.List;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.expression.function.ArrayElemRefExpression;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PArrayDataType;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.util.ByteUtil;
import org.junit.Test;
@@ -31,18 +37,34 @@ import com.google.common.collect.Lists;
public class ArrayConstructorExpressionTest {
+ private static final LiteralExpression CONSTANT_EXPRESSION = LiteralExpression.newConstant(QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
private static final byte[] BYTE_ARRAY1 = new byte[]{1,2,3,4,5};
private static final byte[] BYTE_ARRAY2 = new byte[]{6,7,8};
-
+ private Expression FALSE_EVAL_EXPRESSION = new DelegateExpression(LiteralExpression.newConstant(null)) {
+ @Override
+ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
+ return false;
+ }
+ };
+
+ @Test
+ public void testLeadingNullsForSortableSerialization() throws Exception {
+ helpTestLeadingNulls(PArrayDataType.SORTABLE_SERIALIZATION_VERSION);
+ }
+
@Test
- public void testArraysWithLeadingNulls() throws Exception {
+ public void testLeadingNullsForImmutableSerialization() throws Exception {
+ helpTestLeadingNulls(PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
+ }
+
+ public void helpTestLeadingNulls(byte serializationVersion) throws Exception {
List<Expression> children = Lists.newArrayListWithExpectedSize(4);
LiteralExpression nullExpression = LiteralExpression.newConstant(null);
children.add(nullExpression);
children.add(nullExpression);
children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
- ArrayConstructorExpression arrayConstructorExpression = new ArrayConstructorExpression(children, PVarbinary.INSTANCE, false);
+ ArrayConstructorExpression arrayConstructorExpression = new ArrayConstructorExpression(children, PVarbinary.INSTANCE, false, serializationVersion);
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
ArrayElemRefExpression arrayElemRefExpression = new ArrayElemRefExpression(Lists.<Expression>newArrayList(arrayConstructorExpression));
@@ -60,4 +82,82 @@ public class ArrayConstructorExpressionTest {
arrayElemRefExpression.evaluate(null, ptr);
assertArrayEquals(BYTE_ARRAY2, ptr.copyBytesIfNecessary());
}
+
+ @Test
+ public void testWithExpressionsThatEvaluatetoFalse() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ children.add(CONSTANT_EXPRESSION);
+ children.add(FALSE_EVAL_EXPRESSION);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
+ children.add(FALSE_EVAL_EXPRESSION);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
+ ArrayConstructorExpression arrayConstructorExpression = new ArrayConstructorExpression(children, PVarbinary.INSTANCE, false, PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+
+ ArrayElemRefExpression arrayElemRefExpression = new ArrayElemRefExpression(Lists.<Expression>newArrayList(arrayConstructorExpression));
+ arrayElemRefExpression.setIndex(1);
+ assertTrue(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptr.copyBytesIfNecessary());
+ arrayElemRefExpression.setIndex(2);
+ assertFalse(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptr.copyBytesIfNecessary());
+ arrayElemRefExpression.setIndex(3);
+ assertTrue(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(BYTE_ARRAY1, ptr.copyBytesIfNecessary());
+ arrayElemRefExpression.setIndex(4);
+ assertFalse(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptr.copyBytesIfNecessary());
+ arrayElemRefExpression.setIndex(5);
+ assertTrue(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(BYTE_ARRAY2, ptr.copyBytesIfNecessary());
+ }
+
+ @Test
+ public void testWithMaxOffsetLargerThanShortMax() throws Exception {
+ int numElements = Short.MAX_VALUE+2;
+ List<Expression> children = Lists.newArrayListWithExpectedSize(numElements);
+ for (int i=0; i<numElements; ++i) {
+ children.add(CONSTANT_EXPRESSION);
+ }
+ ArrayConstructorExpression arrayConstructorExpression = new ArrayConstructorExpression(children, PVarbinary.INSTANCE, false, PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
+ ArrayElemRefExpression arrayElemRefExpression = new ArrayElemRefExpression(Lists.<Expression>newArrayList(arrayConstructorExpression));
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+
+ arrayElemRefExpression.setIndex(1);
+ assertTrue(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptr.copyBytesIfNecessary());
+
+ arrayElemRefExpression.setIndex(15000);
+ assertTrue(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptr.copyBytesIfNecessary());
+
+ arrayElemRefExpression.setIndex(numElements);
+ assertTrue(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptr.copyBytesIfNecessary());
+ }
+
+ @Test
+ public void testWithMaxOffsetSmallerThanShortMin() throws Exception {
+ int numElements = Short.MAX_VALUE+2;
+ List<Expression> children = Lists.newArrayListWithExpectedSize(numElements);
+ for (int i=1; i<numElements; i+=2) {
+ children.add(CONSTANT_EXPRESSION);
+ children.add(FALSE_EVAL_EXPRESSION);
+ }
+ ArrayConstructorExpression arrayConstructorExpression = new ArrayConstructorExpression(children, PVarbinary.INSTANCE, false, PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
+ ArrayElemRefExpression arrayElemRefExpression = new ArrayElemRefExpression(Lists.<Expression>newArrayList(arrayConstructorExpression));
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+
+ arrayElemRefExpression.setIndex(2);
+ assertFalse(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptr.copyBytesIfNecessary());
+
+ arrayElemRefExpression.setIndex(15000);
+ assertFalse(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptr.copyBytesIfNecessary());
+
+ arrayElemRefExpression.setIndex(numElements);
+ assertFalse(arrayElemRefExpression.evaluate(null, ptr));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptr.copyBytesIfNecessary());
+ }
}