You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by vi...@apache.org on 2017/12/27 23:42:04 UTC
phoenix git commit: PHOENIX-4382 Immutable table
SINGLE_CELL_ARRAY_WITH_OFFSETS values starting with separator byte return
null in query results
Repository: phoenix
Updated Branches:
refs/heads/5.x-HBase-2.0 b4ccdd2c9 -> e43202f3e
PHOENIX-4382 Immutable table SINGLE_CELL_ARRAY_WITH_OFFSETS values starting with separator byte return null in query results
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e43202f3
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e43202f3
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e43202f3
Branch: refs/heads/5.x-HBase-2.0
Commit: e43202f3e957097d6a6672499e15373d60ab838c
Parents: b4ccdd2
Author: Vincent Poon <vi...@apache.org>
Authored: Wed Dec 27 15:41:41 2017 -0800
Committer: Vincent Poon <vi...@apache.org>
Committed: Wed Dec 27 15:41:58 2017 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/StoreNullsIT.java | 6 +-
.../phoenix/end2end/UpsertBigValuesIT.java | 64 ++++-
.../phoenix/end2end/index/DropColumnIT.java | 19 +-
.../phoenix/compile/ProjectionCompiler.java | 6 +-
.../expression/SingleCellColumnExpression.java | 35 ++-
.../apache/phoenix/index/IndexMaintainer.java | 2 +-
.../NonAggregateRegionScannerFactory.java | 2 +-
.../org/apache/phoenix/schema/ColumnRef.java | 4 +-
.../java/org/apache/phoenix/schema/PTable.java | 21 +-
.../phoenix/schema/types/PArrayDataType.java | 18 +-
.../schema/types/PArrayDataTypeDecoder.java | 79 +++++-
.../schema/types/PArrayDataTypeEncoder.java | 10 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 3 +-
.../schema/ImmutableStorageSchemeTest.java | 241 +++++++++++++++++--
14 files changed, 452 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java
index 63f127c..0f23dac 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java
@@ -148,7 +148,11 @@ public class StoreNullsIT extends ParallelStatsDisabledIT {
byte[] qualifier = table.getImmutableStorageScheme()== ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES : nameColumn.getColumnQualifierBytes();
assertTrue(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, qualifier));
assertTrue(rs.size() == 2); // 2 because it also includes the empty key value column
- KeyValueColumnExpression colExpression = table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? new SingleCellColumnExpression(nameColumn, "NAME", table.getEncodingScheme()) : new KeyValueColumnExpression(nameColumn);
+ KeyValueColumnExpression colExpression =
+ table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS
+ ? new SingleCellColumnExpression(nameColumn, "NAME",
+ table.getEncodingScheme(), table.getImmutableStorageScheme())
+ : new KeyValueColumnExpression(nameColumn);
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
colExpression.evaluate(new ResultTuple(rs), ptr);
assertEquals(new ImmutableBytesPtr(PVarchar.INSTANCE.toBytes("v1")), ptr);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java
index ceb76d0..9842434 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertBigValuesIT.java
@@ -25,10 +25,16 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.util.Arrays;
+import java.util.List;
import java.util.Properties;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PSmallint;
import org.junit.Test;
+import com.google.common.collect.Lists;
public class UpsertBigValuesIT extends ParallelStatsDisabledIT {
@@ -374,4 +380,60 @@ public class UpsertBigValuesIT extends ParallelStatsDisabledIT {
*/
conn.close();
}
-}
+
+ @Test
+ public void testShort() throws Exception {
+ List<Short> testData =
+ Arrays.asList(Short.MIN_VALUE, Short.MAX_VALUE, (short) (Short.MIN_VALUE + 1),
+ (short) (Short.MAX_VALUE - 1), (short) 0, (short) 1, (short) -1);
+ testValues(false, PSmallint.INSTANCE, testData);
+ testValues(true, PSmallint.INSTANCE, testData);
+ }
+
+ @Test
+ public void testBigInt() throws Exception {
+ List<Long> testData =
+ Arrays.asList(Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE + 1L,
+ Long.MAX_VALUE - 1L, 0L, 1L, -1L);
+ testValues(false, PLong.INSTANCE, testData);
+ testValues(true, PLong.INSTANCE, testData);
+ }
+
+ private <T extends Number> void testValues(boolean immutable, PDataType<?> dataType, List<T> testData) throws Exception {
+ String tableName = generateUniqueName();
+ String ddl =
+ String.format("CREATE %s TABLE %s (K INTEGER PRIMARY KEY, V1 %s)",
+ immutable ? "IMMUTABLE" : "", tableName, dataType.getSqlTypeName());
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute(ddl);
+ String upsert = "UPSERT INTO " + tableName + " VALUES(?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ int id = 1;
+ for (T testVal : testData) {
+ stmt.setInt(1, id++);
+ stmt.setObject(2, testVal, dataType.getSqlType());
+ stmt.execute();
+ }
+ conn.commit();
+ String query = String.format("SELECT K,V1 FROM %s ORDER BY K ASC", tableName);
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ int index = 0;
+ boolean failed = false;
+ List<String> errors = Lists.newArrayList();
+ while (rs.next()) {
+ Number resultVal = rs.getObject(2, testData.get(0).getClass());
+ T testVal = testData.get(index++);
+ if (!testVal.equals(resultVal)) {
+ errors.add(String.format("[expected=%s actual=%s] ",
+ testVal, resultVal));
+ failed = true;
+ }
+ }
+ String errorMsg =
+ String.format("Data in table didn't match input: immutable=%s, dataType=%s, %s",
+ immutable, dataType.getSqlTypeName(), errors);
+ assertFalse(errorMsg, failed);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java
index db445f1..5e2b794 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropColumnIT.java
@@ -254,7 +254,9 @@ public class DropColumnIT extends ParallelStatsDisabledIT {
assertNotNull(result);
byte[] colValue;
if (!mutable && columnEncoded) {
- KeyValueColumnExpression colExpression = new SingleCellColumnExpression(dataColumn, "V2", dataTable.getEncodingScheme());
+ KeyValueColumnExpression colExpression =
+ new SingleCellColumnExpression(dataColumn, "V2", dataTable.getEncodingScheme(),
+ dataTable.getImmutableStorageScheme());
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
colExpression.evaluate(new ResultTuple(result), ptr);
colValue = ptr.copyBytesIfNecessary();
@@ -273,7 +275,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT {
result = results.next();
assertNotNull(result);
if (!mutable && columnEncoded) {
- KeyValueColumnExpression colExpression = new SingleCellColumnExpression(glovalIndexCol, "0:V2", globalIndexTable.getEncodingScheme());
+ KeyValueColumnExpression colExpression =
+ new SingleCellColumnExpression(glovalIndexCol, "0:V2",
+ globalIndexTable.getEncodingScheme(),
+ globalIndexTable.getImmutableStorageScheme());
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
colExpression.evaluate(new ResultTuple(result), ptr);
colValue = ptr.copyBytesIfNecessary();
@@ -293,7 +298,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT {
result = results.next();
assertNotNull(result);
if (!mutable && columnEncoded) {
- KeyValueColumnExpression colExpression = new SingleCellColumnExpression(localIndexCol, "0:V2", localIndexTable.getEncodingScheme());
+ KeyValueColumnExpression colExpression =
+ new SingleCellColumnExpression(localIndexCol, "0:V2",
+ localIndexTable.getEncodingScheme(),
+ localIndexTable.getImmutableStorageScheme());
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
assertTrue(colExpression.evaluate(new ResultTuple(result), ptr));
colValue = ptr.copyBytesIfNecessary();
@@ -387,7 +395,10 @@ public class DropColumnIT extends ParallelStatsDisabledIT {
PColumn localIndexCol = localIndex2.getColumnForColumnName(indexColumnName);
byte[] colValue;
if (!mutable && columnEncoded) {
- KeyValueColumnExpression colExpression = new SingleCellColumnExpression(localIndexCol, indexColumnName, localIndex2.getEncodingScheme());
+ KeyValueColumnExpression colExpression =
+ new SingleCellColumnExpression(localIndexCol, indexColumnName,
+ localIndex2.getEncodingScheme(),
+ localIndex2.getImmutableStorageScheme());
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
colExpression.evaluate(new ResultTuple(result), ptr);
colValue = ptr.copyBytesIfNecessary();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
index a147882..f85b5a8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
@@ -668,7 +668,11 @@ public class ProjectionCompiler {
PTable table = context.getCurrentTable().getTable();
KeyValueColumnExpression keyValueColumnExpression;
if (table.getImmutableStorageScheme() != ImmutableStorageScheme.ONE_CELL_PER_COLUMN) {
- keyValueColumnExpression = new SingleCellColumnExpression(col, col.getName().getString(), table.getEncodingScheme());
+ keyValueColumnExpression =
+ new SingleCellColumnExpression(col,
+ col.getName().getString(),
+ table.getEncodingScheme(),
+ table.getImmutableStorageScheme());
} else {
keyValueColumnExpression = new KeyValueColumnExpression(col);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java
index 8c1e0b6..e46e1fa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java
@@ -25,7 +25,6 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;
import org.apache.phoenix.compile.CreateTableCompiler.ViewWhereExpressionVisitor;
import org.apache.phoenix.expression.visitor.ExpressionVisitor;
@@ -54,12 +53,19 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression {
private String arrayColDisplayName;
private KeyValueColumnExpression keyValueColumnExpression;
private QualifierEncodingScheme encodingScheme;
-
+ private ImmutableStorageScheme immutableStorageScheme;
+
public SingleCellColumnExpression() {
}
-
- public SingleCellColumnExpression(PDatum column, byte[] cf, byte[] cq, QualifierEncodingScheme encodingScheme) {
+
+ public SingleCellColumnExpression(ImmutableStorageScheme immutableStorageScheme) {
+ this.immutableStorageScheme = immutableStorageScheme;
+ }
+
+ public SingleCellColumnExpression(PDatum column, byte[] cf, byte[] cq,
+ QualifierEncodingScheme encodingScheme, ImmutableStorageScheme immutableStorageScheme) {
super(column, cf, SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES);
+ this.immutableStorageScheme = immutableStorageScheme;
Preconditions.checkNotNull(encodingScheme);
Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS);
this.decodedColumnQualifier = encodingScheme.decode(cq);
@@ -67,8 +73,9 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression {
setKeyValueExpression();
}
- public SingleCellColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme) {
+ public SingleCellColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme, ImmutableStorageScheme immutableStorageScheme) {
super(column, column.getFamilyName().getBytes(), SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES);
+ this.immutableStorageScheme = immutableStorageScheme;
Preconditions.checkNotNull(encodingScheme);
Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS);
this.arrayColDisplayName = displayName;
@@ -86,8 +93,6 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression {
}
// the first position is reserved and we offset maxEncodedColumnQualifier by ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier)
int index = decodedColumnQualifier-QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+1;
- byte serializedImmutableStorageScheme = ptr.get()[ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_BYTE];
- ImmutableStorageScheme immutableStorageScheme = ImmutableStorageScheme.fromSerializedValue(serializedImmutableStorageScheme);
// Given a ptr to the entire array, set ptr to point to a particular element within that array
ColumnValueDecoder encoderDecoder = immutableStorageScheme.getDecoder();
return encoderDecoder.decode(ptr, index);
@@ -97,7 +102,18 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression {
public void readFields(DataInput input) throws IOException {
super.readFields(input);
this.decodedColumnQualifier = WritableUtils.readVInt(input);
- this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)];
+ int serializedEncodingScheme = WritableUtils.readVInt(input);
+ // prior to PHOENIX-4432 we weren't writing out the immutableStorageScheme in write(),
+ // so we use the decodedColumnQualifier sign to determine whether it's there
+ if (Integer.signum(serializedEncodingScheme) == -1) {
+ this.immutableStorageScheme =
+ ImmutableStorageScheme
+ .fromSerializedValue((byte) WritableUtils.readVInt(input));
+ serializedEncodingScheme = -serializedEncodingScheme;
+ } else {
+ this.immutableStorageScheme = ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
+ }
+ this.encodingScheme = QualifierEncodingScheme.values()[serializedEncodingScheme];
setKeyValueExpression();
}
@@ -105,7 +121,8 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression {
public void write(DataOutput output) throws IOException {
super.write(output);
WritableUtils.writeVInt(output, decodedColumnQualifier);
- WritableUtils.writeVInt(output, encodingScheme.ordinal());
+ WritableUtils.writeVInt(output, -encodingScheme.ordinal()); //negative since PHOENIX-4432
+ WritableUtils.writeVInt(output, immutableStorageScheme.getSerializedMetadataValue());
}
public KeyValueColumnExpression getKeyValueExpression() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index 887a04c..50ef18e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -1010,7 +1010,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
public PDataType getDataType() {
return null;
}
- }, dataColRef.getFamily(), dataColRef.getQualifier(), encodingScheme);
+ }, dataColRef.getFamily(), dataColRef.getQualifier(), encodingScheme, immutableStorageScheme);
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
expression.evaluate(new ValueGetterTuple(valueGetter, ts), ptr);
byte[] value = ptr.copyBytesIfNecessary();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
index c78280d..41d60bf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
@@ -209,7 +209,7 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
int arrayKVRefSize = WritableUtils.readVInt(input);
for (int i = 0; i < arrayKVRefSize; i++) {
PTable.ImmutableStorageScheme scheme = EncodedColumnsUtil.getImmutableStorageScheme(scan);
- KeyValueColumnExpression kvExp = scheme != PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN ? new SingleCellColumnExpression()
+ KeyValueColumnExpression kvExp = scheme != PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN ? new SingleCellColumnExpression(scheme)
: new KeyValueColumnExpression();
kvExp.readFields(input);
arrayKVRefs.add(kvExp);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
index c73b860..fa3156d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java
@@ -126,7 +126,9 @@ public class ColumnRef {
}
Expression expression = table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ?
- new SingleCellColumnExpression(column, displayName, table.getEncodingScheme()) : new KeyValueColumnExpression(column, displayName);
+ new SingleCellColumnExpression(column, displayName,
+ table.getEncodingScheme(), table.getImmutableStorageScheme())
+ : new KeyValueColumnExpression(column, displayName);
if (column.getExpressionStr() != null) {
String url = PhoenixRuntime.JDBC_PROTOCOL
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index ec931b7..7e186ad 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -190,14 +190,14 @@ public interface PTable extends PMetaDataEntity {
}
},
// stores a single cell per column family that contains all serialized column values
- SINGLE_CELL_ARRAY_WITH_OFFSETS((byte)2) {
+ SINGLE_CELL_ARRAY_WITH_OFFSETS((byte)2, PArrayDataType.IMMUTABLE_SERIALIZATION_V2) {
@Override
public ColumnValueEncoder getEncoder(int numElements) {
PDataType type = PVarbinary.INSTANCE;
int estimatedSize = PArrayDataType.estimateSize(numElements, type);
TrustedByteArrayOutputStream byteStream = new TrustedByteArrayOutputStream(estimatedSize);
DataOutputStream oStream = new DataOutputStream(byteStream);
- return new PArrayDataTypeEncoder(byteStream, oStream, numElements, type, SortOrder.ASC, false, PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
+ return new PArrayDataTypeEncoder(byteStream, oStream, numElements, type, SortOrder.ASC, false, getSerializationVersion());
}
@Override
@@ -207,15 +207,30 @@ public interface PTable extends PMetaDataEntity {
};
private final byte serializedValue;
-
+ private byte serializationVersion;
+
private ImmutableStorageScheme(byte serializedValue) {
this.serializedValue = serializedValue;
}
+ private ImmutableStorageScheme(byte serializedValue, byte serializationVersion) {
+ this.serializedValue = serializedValue;
+ this.serializationVersion = serializationVersion;
+ }
+
public byte getSerializedMetadataValue() {
return this.serializedValue;
}
+ public byte getSerializationVersion() {
+ return this.serializationVersion;
+ }
+
+ @VisibleForTesting
+ void setSerializationVersion(byte serializationVersion) {
+ this.serializationVersion = serializationVersion;
+ }
+
public static ImmutableStorageScheme fromSerializedValue(byte serializedValue) {
if (serializedValue < 1 || serializedValue > ImmutableStorageScheme.values().length) {
return null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
index 7d742e2..162b235 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
@@ -75,7 +75,12 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
// array serialization format where bytes can be used as part of the row key
public static final byte SORTABLE_SERIALIZATION_VERSION = 1;
// array serialization format where bytes are immutable (does not support prepend/append or sorting)
+ @Deprecated
public static final byte IMMUTABLE_SERIALIZATION_VERSION = 2;
+ // array serialization format where bytes are immutable (does not support prepend/append or sorting)
+ // differs from V1 in that nulls are not serialized
+ // we rely only on offsets to determine the presence of nulls
+ public static final byte IMMUTABLE_SERIALIZATION_V2 = 3;
protected PArrayDataType(String sqlTypeName, int sqlType, Class clazz, PDataCodec codec, int ordinal) {
super(sqlTypeName, sqlType, clazz, codec, ordinal);
@@ -217,7 +222,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
public static boolean useShortForOffsetArray(int maxoffset, byte serializationVersion) {
- if (serializationVersion == IMMUTABLE_SERIALIZATION_VERSION) {
+ if (serializationVersion == IMMUTABLE_SERIALIZATION_VERSION || serializationVersion == IMMUTABLE_SERIALIZATION_V2) {
return (maxoffset <= Short.MAX_VALUE && maxoffset >= Short.MIN_VALUE );
}
// If the max offset is less than Short.MAX_VALUE then offset array can use short
@@ -383,7 +388,10 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
int offset;
if (useShort) {
offset = indexOffset + (Bytes.SIZEOF_SHORT * arrayIndex);
- return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT) + (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION ? 0 : Short.MAX_VALUE);
+ return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT)
+ + (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION
+ || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2 ? 0
+ : Short.MAX_VALUE);
} else {
offset = indexOffset + (Bytes.SIZEOF_INT * arrayIndex);
return Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
@@ -964,7 +972,11 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
} else {
for (int pos : offsetPos) {
- short val = serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION ? (short)pos : (short)(pos - Short.MAX_VALUE);
+ short val =
+ serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION
+ || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2
+ ? (short) pos
+ : (short) (pos - Short.MAX_VALUE);
Bytes.putShort(offsetArr, off, val);
off += Bytes.SIZEOF_SHORT;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java
index 7a6ea91..22fa46c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeDecoder.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.ColumnValueDecoder;
+import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.ByteUtil;
@@ -78,13 +79,45 @@ public class PArrayDataTypeDecoder implements ColumnValueDecoder {
}
int elementLength = 0;
if (arrayIndex == (noOfElements - 1)) {
- int separatorBytes = serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 3 : 0;
- elementLength = (bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : indexOffset
- - (currOffset + initPos) - separatorBytes;
+ // in the original IMMUTABLE_SERIALIZATION_VERSION (v1), for nulls we store
+ // (separatorByte, #_of_nulls) in the data. Because of the separatorByte, we can't
+ // distinguish between nulls and actual data values that start with the separator
+ // byte. We do a hack here to limit the damage by checking offsets - if the prior
+ // offset had a length of 0, then we know we're storing 2 or more nulls. However, we
+ // still can't fix the case distinguishing a single null from a short value. There
+ // are two kinds of separatorByte, so the results will be potentially incorrect for
+ // 2 short values that correspond to (separatorByte, 1)
+ if (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION) {
+ elementLength = indexOffset - (currOffset + initPos);
+ if (isNullValue(arrayIndex, bytes, initPos, serializationVersion, useShort, indexOffset, currOffset, elementLength)) {
+ elementLength = 0;
+ }
+ } else {
+ int separatorBytes = serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 3 : 0;
+ elementLength = isSeparatorByte(bytes, initPos, currOffset) && serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 0 : indexOffset
+ - (currOffset + initPos) - separatorBytes;
+ }
} else {
- int separatorByte = serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION ? 1 : 0;
- elementLength = (bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE) ? 0 : PArrayDataType.getOffset(bytes,
- arrayIndex + 1, useShort, indexOffset, serializationVersion) - currOffset - separatorByte;
+ if (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION) {
+ elementLength = PArrayDataType.getOffset(bytes, arrayIndex + 1,
+ useShort, indexOffset, serializationVersion)
+ - currOffset;
+ if (isNullValue(arrayIndex, bytes, initPos, serializationVersion, useShort, indexOffset, currOffset, elementLength)) {
+ elementLength = 0;
+ }
+ } else {
+ int separatorByte =
+ serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION
+ ? 1
+ : 0;
+ elementLength =
+ isSeparatorByte(bytes, initPos, currOffset)
+ && serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION
+ ? 0
+ : PArrayDataType.getOffset(bytes, arrayIndex + 1,
+ useShort, indexOffset, serializationVersion)
+ - currOffset - separatorByte;
+ }
}
ptr.set(bytes, currOffset + initPos, elementLength);
} else {
@@ -99,4 +132,38 @@ public class PArrayDataTypeDecoder implements ColumnValueDecoder {
return true;
}
+ // returns true if the prior element in the array is a null
+ private static boolean isNullValue(int arrayIndex, byte[] bytes, int initPos,
+ byte serializationVersion, boolean useShort, int indexOffset, int currOffset,
+ int elementLength) {
+ if (isSeparatorByte(bytes, initPos, currOffset)) {
+ if (isPriorValueZeroLength(arrayIndex, bytes,
+ serializationVersion, useShort, indexOffset, currOffset)) {
+ return true;
+ } else {
+ // if there's no prior null, there can be at most 1 null
+ if (elementLength == 2) {
+ // nullByte calculation comes from the encoding of one null
+ // see PArrayDataType#serializeNulls
+ byte nullByte = SortOrder.invert((byte)(0));
+ if (bytes[initPos+currOffset+1] == nullByte) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ // checks prior value length by subtracting offset of the previous item from the current offset
+ private static boolean isPriorValueZeroLength(int arrayIndex, byte[] bytes, byte serializationVersion,
+ boolean useShort, int indexOffset, int currOffset) {
+ return arrayIndex > 0 && currOffset - PArrayDataType.getOffset(bytes, arrayIndex - 1,
+ useShort, indexOffset, serializationVersion) == 0;
+ }
+
+ private static boolean isSeparatorByte(byte[] bytes, int initPos, int currOffset) {
+ return bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE || bytes[currOffset + initPos] == QueryConstants.DESC_SEPARATOR_BYTE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java
index 3dad6c3..7467981 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataTypeEncoder.java
@@ -92,7 +92,9 @@ public class PArrayDataTypeEncoder implements ColumnValueEncoder {
// used to represent the absence of a value
@Override
public void appendAbsentValue() {
- if (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION && !baseType.isFixedWidth()) {
+ if ((serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION
+ || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2)
+ && !baseType.isFixedWidth()) {
offsetPos.add(-byteStream.size());
nulls++;
}
@@ -125,7 +127,11 @@ public class PArrayDataTypeEncoder implements ColumnValueEncoder {
offsetPos.add(byteStream.size());
nulls++;
} else {
- nulls = PArrayDataType.serializeNulls(oStream, nulls);
+ // we don't serialize nulls for IMMUTABLE_SERIALIZATION_V2
+ if (serializationVersion == PArrayDataType.SORTABLE_SERIALIZATION_VERSION
+ || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION) {
+ nulls = PArrayDataType.serializeNulls(oStream, nulls);
+ }
offsetPos.add(byteStream.size());
if (sortOrder == SortOrder.DESC) {
SortOrder.invert(bytes, offset, bytes, offset, len);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index f1e5582..6925ac6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -468,13 +468,14 @@ public class IndexUtil {
KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan);
boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL) != null;
QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);
+ ImmutableStorageScheme immutableStorageScheme = EncodedColumnsUtil.getImmutableStorageScheme(scan);
Expression[] colExpressions = storeColsInSingleCell ? new SingleCellColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length];
for (int i = 0; i < dataColumns.length; i++) {
byte[] family = dataColumns[i].getFamily();
byte[] qualifier = dataColumns[i].getQualifier();
Field field = keyValueSchema.getField(i);
Expression dataColumnExpr =
- storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme)
+ storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme, immutableStorageScheme)
: new KeyValueColumnExpression(field, family, qualifier);
colExpressions[i] = dataColumnExpr;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e43202f3/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java
index d8c5cdb..3d1b176 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java
@@ -17,9 +17,15 @@
*/
package org.apache.phoenix.schema;
+import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
+import static org.apache.phoenix.schema.types.PArrayDataType.IMMUTABLE_SERIALIZATION_V2;
+import static org.apache.phoenix.schema.types.PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION;
+import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.util.Arrays;
import java.util.List;
@@ -33,6 +39,11 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PSmallint;
+import org.apache.phoenix.schema.types.PTinyint;
+import org.apache.phoenix.schema.types.PUnsignedTinyint;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.util.ByteUtil;
import org.junit.Test;
@@ -55,16 +66,22 @@ public class ImmutableStorageSchemeTest {
}
};
private ImmutableStorageScheme immutableStorageScheme;
+ byte serializationVersion;
- @Parameters(name="ImmutableStorageSchemeTest_immutableStorageScheme={0}}") // name is used by failsafe as file name in reports
- public static ImmutableStorageScheme[] data() {
- ImmutableStorageScheme[] values = ImmutableStorageScheme.values();
- // skip ONE_CELL_PER_COLUMN
- return Arrays.copyOfRange(values, 1, values.length);
+ @Parameters(name="ImmutableStorageSchemeTest_immutableStorageScheme={0},serializationVersion={1}}") // name is used by failsafe as file name in reports
+ public static List<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { SINGLE_CELL_ARRAY_WITH_OFFSETS,
+ IMMUTABLE_SERIALIZATION_VERSION },
+ { SINGLE_CELL_ARRAY_WITH_OFFSETS,
+ IMMUTABLE_SERIALIZATION_V2 }
+ });
}
- public ImmutableStorageSchemeTest(ImmutableStorageScheme immutableStorageScheme) {
+ public ImmutableStorageSchemeTest(ImmutableStorageScheme immutableStorageScheme, byte serializationVersion) {
this.immutableStorageScheme = immutableStorageScheme;
+ this.immutableStorageScheme.setSerializationVersion(serializationVersion);
+ this.serializationVersion = serializationVersion;
}
@Test
@@ -75,9 +92,7 @@ public class ImmutableStorageSchemeTest {
children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
children.add(FALSE_EVAL_EXPRESSION);
children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
- SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children);
- ImmutableBytesPtr ptr = new ImmutableBytesPtr();
- singleCellConstructorExpression.evaluate(null, ptr);
+ ImmutableBytesPtr ptr = evaluate(children);
ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
@@ -160,23 +175,201 @@ public class ImmutableStorageSchemeTest {
children.add(nullExpression);
children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
- SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children);
+ ImmutableBytesPtr ptr = evaluate(children);
+
+ assertDecodedContents(ptr, new byte[][] {EMPTY_BYTE_ARRAY, EMPTY_BYTE_ARRAY, BYTE_ARRAY1, BYTE_ARRAY2});
+ }
+
+ @Test
+ public void testTrailingNulls() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ LiteralExpression nullExpression = LiteralExpression.newConstant(null);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
+ children.add(nullExpression);
+ children.add(nullExpression);
+ ImmutableBytesPtr ptr = evaluate(children);
+
+ assertDecodedContents(ptr, new byte[][] {BYTE_ARRAY1, BYTE_ARRAY2, EMPTY_BYTE_ARRAY, EMPTY_BYTE_ARRAY});
+ }
+
+ @Test
+ public void testManyNulls() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ LiteralExpression nullExpression = LiteralExpression.newConstant(null);
+ byte[][] testData = new byte[300][];
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
+ testData[0] = BYTE_ARRAY1;
+ for (int i = 1; i < testData.length - 1; i++) {
+ children.add(nullExpression);
+ testData[i] = EMPTY_BYTE_ARRAY;
+ }
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
+ testData[299] = BYTE_ARRAY2;
+ ImmutableBytesPtr ptr = evaluate(children);
+
+ assertDecodedContents(ptr, testData);
+ }
+
+ @Test
+ public void testSingleLeadingTrailingNull() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ LiteralExpression nullExpression = LiteralExpression.newConstant(null);
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
+ children.add(nullExpression);
+ ImmutableBytesPtr ptr = evaluate(children);
+
+ assertDecodedContents(ptr,
+ new byte[][] { EMPTY_BYTE_ARRAY, BYTE_ARRAY1, EMPTY_BYTE_ARRAY });
+ }
+
+ @Test
+ public void testSingleMiddleNull() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ LiteralExpression nullExpression = LiteralExpression.newConstant(null);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
+ ImmutableBytesPtr ptr = evaluate(children);
+
+ assertDecodedContents(ptr, new byte[][] { BYTE_ARRAY1, EMPTY_BYTE_ARRAY, BYTE_ARRAY2 });
+ }
+
+ @Test
+ public void testAllShortValues() throws Exception {
+ int curr = Short.MIN_VALUE;
+ List<Expression> children = Lists.newArrayListWithExpectedSize(1);
+ List<Integer> failedValues = Lists.newArrayList();
+ while (curr <= Short.MAX_VALUE) {
+ children.add(LiteralExpression.newConstant(curr, PSmallint.INSTANCE));
+ ImmutableBytesPtr ptr = evaluate(children);
+ ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
+ assertTrue(decoder.decode(ptr, 0));
+ if (ptr.getLength() == 0) {
+ failedValues.add(curr);
+ } else {
+ if (curr != PSmallint.INSTANCE.getCodec().decodeShort(ptr.copyBytesIfNecessary(), 0,
+ SortOrder.ASC)) {
+ failedValues.add(curr);
+ }
+ }
+ children.remove(0);
+ curr++;
+ }
+ // in v1, we can't distinguish a null from two short values
+ if (serializationVersion == IMMUTABLE_SERIALIZATION_VERSION) {
+ assertTrue(failedValues.size() + " values were not properly decoded: " + failedValues,
+ failedValues.size() == 2);
+ } else {
+ assertTrue(failedValues.size() + " values were not properly decoded: " + failedValues,
+ failedValues.size() == 0);
+ }
+ }
+
+ @Test
+ public void testSingleByteValues() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ LiteralExpression nullExpression = LiteralExpression.newConstant(null);
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant((byte) -128, PTinyint.INSTANCE));
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant((byte) 0, PUnsignedTinyint.INSTANCE));
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant((byte) 127, PUnsignedTinyint.INSTANCE));
+ ImmutableBytesPtr ptr = evaluate(children);
+
+ assertNullAtIndex(ptr, 0);
+ assertValueAtIndex(ptr, 1, (byte) -128, PTinyint.INSTANCE);
+ assertNullAtIndex(ptr, 2);
+ assertValueAtIndex(ptr, 3, (byte) 0, PUnsignedTinyint.INSTANCE);
+ assertNullAtIndex(ptr, 4);
+ assertValueAtIndex(ptr, 5, (byte) 127, PUnsignedTinyint.INSTANCE);
+ }
+
+ @Test
+ public void testSeparatorByteValues() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ LiteralExpression nullExpression = LiteralExpression.newConstant(null);
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant((short) -32513, PSmallint.INSTANCE));
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant((short) 32767, PSmallint.INSTANCE));
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant(Integer.MAX_VALUE, PInteger.INSTANCE));
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant(Integer.MIN_VALUE, PInteger.INSTANCE));
+ // see if we can differentiate two nulls and {separatorByte, 2}
+ children.add(nullExpression);
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant((short) -32514, PSmallint.INSTANCE));
+
+ ImmutableBytesPtr ptr = evaluate(children);
+
+ assertNullAtIndex(ptr, 0);
+ try {
+ assertValueAtIndex(ptr, 1, (short) -32513, PSmallint.INSTANCE);
+ } catch (Exception e) {
+ if (serializationVersion != IMMUTABLE_SERIALIZATION_VERSION) {
+ fail("Failed on exception " + e);
+ }
+ }
+ assertNullAtIndex(ptr, 2);
+ try {
+ assertValueAtIndex(ptr, 3, (short) 32767, PSmallint.INSTANCE);
+ } catch (Exception e) {
+ if (serializationVersion != IMMUTABLE_SERIALIZATION_VERSION) {
+ fail("Failed on exception " + e);
+ }
+ }
+ assertNullAtIndex(ptr, 4);
+ assertValueAtIndex(ptr, 5, Integer.MAX_VALUE, PInteger.INSTANCE);
+ assertNullAtIndex(ptr, 6);
+ assertValueAtIndex(ptr, 7, Integer.MIN_VALUE, PInteger.INSTANCE);
+ assertNullAtIndex(ptr, 8);
+ assertNullAtIndex(ptr, 9);
+ assertValueAtIndex(ptr, 10, (short) -32514, PSmallint.INSTANCE);
+ }
+
+ private void assertNullAtIndex(ImmutableBytesPtr ptr, int index) {
+ assertValueAtIndex(ptr, index, null, null);
+ }
+
+ private void assertValueAtIndex(ImmutableBytesPtr ptr, int index, Object value,
+ PDataType type) {
+ ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
+ ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
+ assertTrue(decoder.decode(ptrCopy, index));
+ if (value == null) {
+ assertArrayEquals(EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
+ return;
+ }
+ Object decoded;
+ if (type.equals(PSmallint.INSTANCE)) {
+ decoded = type.getCodec().decodeShort(ptrCopy.copyBytesIfNecessary(), 0, SortOrder.ASC);
+ } else if (type.equals(PInteger.INSTANCE)) {
+ decoded = type.getCodec().decodeInt(ptrCopy.copyBytesIfNecessary(), 0, SortOrder.ASC);
+ } else { // assume byte for all other types
+ decoded = type.getCodec().decodeByte(ptrCopy.copyBytesIfNecessary(), 0, SortOrder.ASC);
+ }
+ assertEquals(value, decoded);
+ }
+
+ private ImmutableBytesPtr evaluate(List<Expression> children) {
+ SingleCellConstructorExpression singleCellConstructorExpression =
+ new SingleCellConstructorExpression(immutableStorageScheme, children);
ImmutableBytesPtr ptr = new ImmutableBytesPtr();
singleCellConstructorExpression.evaluate(null, ptr);
-
- ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
+ return ptr;
+ }
+
+ private void assertDecodedContents(ImmutableBytesPtr ptr, byte[]... contents) {
ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
- assertTrue(decoder.decode(ptrCopy, 0));
- assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
- ptrCopy = new ImmutableBytesPtr(ptr);
- assertTrue(decoder.decode(ptrCopy, 1));
- assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
- ptrCopy = new ImmutableBytesPtr(ptr);
- assertTrue(decoder.decode(ptrCopy, 2));
- assertArrayEquals(BYTE_ARRAY1, ptrCopy.copyBytesIfNecessary());
- ptrCopy = new ImmutableBytesPtr(ptr);
- assertTrue(decoder.decode(ptrCopy, 3));
- assertArrayEquals(BYTE_ARRAY2, ptrCopy.copyBytesIfNecessary());
+ for (int i = 0; i < contents.length; i++) {
+ ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, i));
+ assertArrayEquals(contents[i], ptrCopy.copyBytesIfNecessary());
+ }
}
-
+
}