You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sa...@apache.org on 2016/09/19 23:46:22 UTC
[8/9] phoenix git commit: PHOENIX-2565 Store data for immutable
tables in single KeyValue (Thomas DSilva)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/920be8fe/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java
index bfbfc54..43f5801 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.util;
import static com.google.common.base.Preconditions.checkArgument;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PColumn;
@@ -29,7 +30,11 @@ import org.apache.phoenix.schema.types.PInteger;
public class EncodedColumnsUtil {
public static boolean usesEncodedColumnNames(PTable table) {
- return table.getStorageScheme() != null && table.getStorageScheme() == StorageScheme.ENCODED_COLUMN_NAMES;
+ return usesEncodedColumnNames(table.getStorageScheme());
+ }
+
+ public static boolean usesEncodedColumnNames(StorageScheme storageSchema) {
+ return storageSchema != null && storageSchema != StorageScheme.NON_ENCODED_COLUMN_NAMES;
}
public static byte[] getEncodedColumnQualifier(PColumn column) {
@@ -41,6 +46,18 @@ public class EncodedColumnsUtil {
public static byte[] getColumnQualifier(PColumn column, PTable table) {
return EncodedColumnsUtil.getColumnQualifier(column, usesEncodedColumnNames(table));
}
+
+ public static void setColumns(PColumn column, PTable table, Scan scan) {
+ if (table.getStorageScheme() == StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL) {
+ // if a table storage scheme is COLUMNS_STORED_IN_SINGLE_CELL set then all columns of a column family are stored in a single cell
+ // (with the qualifier name being same as the family name), just project the column family here
+ // so that we can calculate estimatedByteSize correctly in ProjectionCompiler
+ scan.addFamily(column.getFamilyName().getBytes());
+ }
+ else {
+ scan.addColumn(column.getFamilyName().getBytes(), EncodedColumnsUtil.getColumnQualifier(column, table));
+ }
+ }
public static byte[] getColumnQualifier(PColumn column, boolean encodedColumnName) {
checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/920be8fe/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 76ec772..c6524f7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -29,6 +29,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
@@ -57,7 +58,9 @@ import org.apache.phoenix.compile.WhereCompiler;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState.RowMutationState;
import org.apache.phoenix.execute.TupleProjector;
+import org.apache.phoenix.expression.ArrayColumnExpression;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.KeyValueColumnExpression;
import org.apache.phoenix.expression.RowKeyColumnExpression;
@@ -81,13 +84,16 @@ import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.tuple.ResultTuple;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PBinary;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDecimal;
+import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
@@ -258,10 +264,10 @@ public class IndexUtil {
}
public static List<Mutation> generateIndexData(final PTable table, PTable index,
- List<Mutation> dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
+ final Map<ImmutableBytesPtr, RowMutationState> valuesMap, List<Mutation> dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection)
throws SQLException {
try {
- final ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ final ImmutableBytesPtr ptr = new ImmutableBytesPtr();
IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
for (final Mutation dataMutation : dataMutations) {
@@ -290,20 +296,21 @@ public class IndexUtil {
if (isEmptyKeyValue(table, ref)) {
return null;
}
- Map<byte [], List<Cell>> familyMap = dataMutation.getFamilyCellMap();
byte[] family = ref.getFamily();
- List<Cell> kvs = familyMap.get(family);
- if (kvs == null) {
- return null;
- }
byte[] qualifier = ref.getQualifier();
- for (Cell kv : kvs) {
- if (Bytes.compareTo(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), family, 0, family.length) == 0 &&
- Bytes.compareTo(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(), qualifier, 0, qualifier.length) == 0) {
- ImmutableBytesPtr ptr = new ImmutableBytesPtr();
- kvBuilder.getValueAsPtr(kv, ptr);
- return ptr;
- }
+ RowMutationState rowMutationState = valuesMap.get(ptr);
+ PColumn column = null;
+ try {
+ column = table.getColumnFamily(family).getPColumnForColumnQualifier(qualifier);
+ } catch (ColumnNotFoundException e) {
+ } catch (ColumnFamilyNotFoundException e) {
+ }
+ if (rowMutationState!=null && column!=null) {
+ byte[] value = rowMutationState.getColumnValues().get(column);
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+ ptr.set(value==null ? ByteUtil.EMPTY_BYTE_ARRAY : value);
+ SchemaUtil.padData(table.getName().getString(), column, ptr);
+ return ptr;
}
return null;
}
@@ -316,7 +323,7 @@ public class IndexUtil {
regionStartKey = tableRegionLocation.getRegionInfo().getStartKey();
regionEndkey = tableRegionLocation.getRegionInfo().getEndKey();
}
- indexMutations.add(maintainer.buildUpdateMutation(kvBuilder, valueGetter, ptr, ts, regionStartKey, regionEndkey));
+ indexMutations.add(maintainer.buildUpdateMutation(kvBuilder, valueGetter, ptr, ts, regionStartKey, regionEndkey, true));
}
}
return indexMutations;
@@ -430,13 +437,18 @@ public class IndexUtil {
public static TupleProjector getTupleProjector(Scan scan, ColumnReference[] dataColumns) {
if (dataColumns != null && dataColumns.length != 0) {
KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan);
- KeyValueColumnExpression[] keyValueColumns = new KeyValueColumnExpression[dataColumns.length];
+ boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL)!=null;
+ Expression[] colExpressions = storeColsInSingleCell ? new ArrayColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length];
for (int i = 0; i < dataColumns.length; i++) {
- ColumnReference dataColumn = dataColumns[i];
- KeyValueColumnExpression dataColumnExpr = new KeyValueColumnExpression(keyValueSchema.getField(i), dataColumn.getFamily(), dataColumn.getQualifier());
- keyValueColumns[i] = dataColumnExpr;
+ byte[] family = dataColumns[i].getFamily();
+ byte[] qualifier = dataColumns[i].getQualifier();
+ Field field = keyValueSchema.getField(i);
+ Expression dataColumnExpr =
+ storeColsInSingleCell ? new ArrayColumnExpression(field, family, PInteger.INSTANCE.getCodec().decodeInt(qualifier, 0, SortOrder.getDefault()))
+ : new KeyValueColumnExpression(field, family, qualifier);
+ colExpressions[i] = dataColumnExpr;
}
- return new TupleProjector(keyValueSchema, keyValueColumns);
+ return new TupleProjector(keyValueSchema, colExpressions);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/920be8fe/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index b13f4e3..00ad1ae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -67,6 +67,7 @@ import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
+import org.apache.phoenix.schema.PTable.StorageScheme;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.ValueSchema.Field;
@@ -918,7 +919,8 @@ public class ScanUtil {
}
public static boolean setMinMaxQualifiersOnScan(PTable table) {
- return EncodedColumnsUtil.usesEncodedColumnNames(table) && !table.isTransactional() && !hasDynamicColumns(table);
+ return table.getStorageScheme() != null && table.getStorageScheme() == StorageScheme.ENCODED_COLUMN_NAMES
+ && !table.isTransactional() && !hasDynamicColumns(table);
}
public static boolean hasDynamicColumns(PTable table) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/920be8fe/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index 3071dc0..eb55f38 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -44,8 +44,10 @@ import javax.annotation.Nullable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.exception.DataExceedsCapacityException;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.expression.Expression;
@@ -1043,4 +1045,20 @@ public class SchemaUtil {
}
}
+ /**
+ * Pads the data in ptr by the required amount for fixed width data types
+ */
+ public static void padData(String tableName, PColumn column, ImmutableBytesWritable ptr) {
+ PDataType type = column.getDataType();
+ byte[] byteValue = ptr.get();
+ boolean isNull = type.isNull(byteValue);
+ Integer maxLength = column.getMaxLength();
+ if (!isNull && type.isFixedWidth() && maxLength != null) {
+ if (ptr.getLength() < maxLength) {
+ type.pad(ptr, maxLength, column.getSortOrder());
+ } else if (ptr.getLength() > maxLength) {
+ throw new DataExceedsCapacityException(tableName + "." + column.getName().getString() + " may not exceed " + maxLength + " bytes (" + type.toObject(byteValue) + ")");
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/920be8fe/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
index 5887e5b..abcf0c8 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java
@@ -141,7 +141,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest {
Mutation indexMutation = indexMutations.get(0);
ImmutableBytesWritable indexKeyPtr = new ImmutableBytesWritable(indexMutation.getRow());
ptr.set(rowKeyPtr.get(), rowKeyPtr.getOffset(), rowKeyPtr.getLength());
- byte[] mutablelndexRowKey = im1.buildRowKey(valueGetter, ptr, null, null);
+ byte[] mutablelndexRowKey = im1.buildRowKey(valueGetter, ptr, null, null, false);
byte[] immutableIndexRowKey = indexKeyPtr.copyBytes();
assertArrayEquals(immutableIndexRowKey, mutablelndexRowKey);
for (ColumnReference ref : im1.getCoveredColumns()) {