You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sa...@apache.org on 2016/09/23 17:29:59 UTC
phoenix git commit: Fix column qualifiers for
StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL. Enable encoded column qualifiers
for joins. Bring back qualifier counter per column family. Please enter the
commit message for your changes. Lines starting
Repository: phoenix
Updated Branches:
refs/heads/encodecolumns 29dbf051c -> aaecd5a4d
Fix column qualifiers for StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL.
Enable encoded column qualifiers for joins.
Bring back qualifier counter per column family.
Please enter the commit message for your changes. Lines starting
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/aaecd5a4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/aaecd5a4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/aaecd5a4
Branch: refs/heads/encodecolumns
Commit: aaecd5a4d9c12bcaad2d278b79a87e4ddc888671
Parents: 29dbf05
Author: Samarth <sa...@salesforce.com>
Authored: Fri Sep 23 10:29:02 2016 -0700
Committer: Samarth <sa...@salesforce.com>
Committed: Fri Sep 23 10:29:02 2016 -0700
----------------------------------------------------------------------
.../phoenix/compile/TupleProjectionCompiler.java | 17 ++++-------------
.../phoenix/expression/ArrayColumnExpression.java | 8 +++++---
.../org/apache/phoenix/schema/MetaDataClient.java | 9 ++++-----
.../java/org/apache/phoenix/schema/PTable.java | 15 +++++++++++++++
.../java/org/apache/phoenix/schema/PTableImpl.java | 2 +-
.../tuple/EncodedColumnQualiferCellsList.java | 4 ++--
.../java/org/apache/phoenix/util/ScanUtil.java | 2 +-
.../apache/phoenix/execute/CorrelatePlanTest.java | 4 +++-
.../execute/LiteralResultIteratorPlanTest.java | 4 +++-
9 files changed, 38 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 3898969..9f7de36 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.parse.AliasedNode;
import org.apache.phoenix.parse.ColumnParseNode;
import org.apache.phoenix.parse.FamilyWildcardParseNode;
@@ -168,8 +167,6 @@ public class TupleProjectionCompiler {
boolean hasSaltingColumn = retainPKColumns && table.getBucketNum() != null;
List<PColumn> projectedColumns = new ArrayList<PColumn>();
int position = hasSaltingColumn ? 1 : 0;
- EncodedCQCounter cqCounter = null;
- String valueColFamilyName = Bytes.toString(VALUE_COLUMN_FAMILY);
for (int i = position; i < sourceColumnRefs.size(); i++) {
ColumnRef sourceColumnRef = sourceColumnRefs.get(i);
PColumn sourceColumn = sourceColumnRef.getColumn();
@@ -182,18 +179,12 @@ public class TupleProjectionCompiler {
retainPKColumns && SchemaUtil.isPKColumn(sourceColumn) ?
null : PNameFactory.newName(VALUE_COLUMN_FAMILY),
position++, sourceColumn.isNullable(), sourceColumnRef);
- if (EncodedColumnsUtil.hasEncodedColumnName(sourceColumn)) {
- if (cqCounter == null) {
- cqCounter = new EncodedCQCounter();
- } else {
- cqCounter.increment(valueColFamilyName);
- }
- }
projectedColumns.add(column);
}
- StorageScheme storageScheme = StorageScheme.NON_ENCODED_COLUMN_NAMES;
- if (cqCounter != null) {
- storageScheme = StorageScheme.ENCODED_COLUMN_NAMES;
+ StorageScheme storageScheme = table.getStorageScheme();
+ EncodedCQCounter cqCounter = EncodedCQCounter.NULL_COUNTER;
+ if (storageScheme != StorageScheme.NON_ENCODED_COLUMN_NAMES) {
+ cqCounter = EncodedCQCounter.copy(table.getEncodedCQCounter());
}
return PTableImpl.makePTable(table.getTenantId(), PROJECTED_TABLE_SCHEMA, table.getName(), PTableType.PROJECTED,
null, table.getTimeStamp(), table.getSequenceNumber(), table.getPKName(),
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
index aaa9f75..7310e59 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
@@ -30,6 +30,7 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PArrayDataType;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.util.EncodedColumnsUtil;
/**
*
@@ -57,10 +58,11 @@ public class ArrayColumnExpression extends ColumnExpression {
public ArrayColumnExpression(PColumn column, String displayName, boolean encodedColumnName) {
super(column);
- // array indexes are 1-based
- this.index = column.getEncodedColumnQualifier()+1;
+ // array indexes are 1-based TODO: samarth think about the case when the encodedcolumn qualifier is null. Probably best to add a check here for encodedcolumnname to be true
+ this.index = column.getEncodedColumnQualifier() + 1;
byte[] cf = column.getFamilyName().getBytes();
- this.arrayExpression = new KeyValueColumnExpression(column, cf, cf);
+ byte[] cq = EncodedColumnsUtil.getEncodedColumnQualifier(column);
+ this.arrayExpression = new KeyValueColumnExpression(column, cf, cq);
this.origKVExpression = new KeyValueColumnExpression(column, displayName, encodedColumnName);
this.displayName = displayName;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 5f160cc..beaa376 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -2052,8 +2052,8 @@ public class MetaDataClient {
* in the client cache. If the phoenix table already doesn't exist then the non-encoded column qualifier scheme works
* because we cannot control the column qualifiers that were used when populating the hbase table.
*/
- //TODO: samarth these checks need to be changed for local indexes. Hate having all of these special cases for local indexes.
- // It is making the code unmaintainable.
+ //TODO: samarth these checks for whether table exists need to be changed for local indexes. Hate having all of these special cases for local indexes.
+ // It is making the code unmaintainable. For local indexes, the physical table already exists.
byte[] tableNameBytes = SchemaUtil.getTableNameAsBytes(schemaName, tableName);
boolean tableExists = true;
try (HBaseAdmin admin = connection.getQueryServices().getAdmin()) {
@@ -2072,7 +2072,6 @@ public class MetaDataClient {
} else if (tableExists) {
storageScheme = StorageScheme.NON_ENCODED_COLUMN_NAMES;
} else if (isImmutableRows) {
-// storageScheme = StorageScheme.NON_ENCODED_COLUMN_NAMES;
storageScheme = StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL;
// since we are storing all columns of a column family in a single key value we can't use deletes to store nulls
storeNulls = true;
@@ -2147,14 +2146,14 @@ public class MetaDataClient {
}
}
- if (storageScheme == StorageScheme.ENCODED_COLUMN_NAMES) {
+ if (EncodedColumnsUtil.usesEncodedColumnNames(storageScheme)) {
// Store the encoded column counter for phoenix entities that have their own hbase
// tables i.e. base tables and indexes.
String schemaNameToUse = tableType == VIEW ? viewPhysicalTable.getSchemaName().getString() : schemaName;
String tableNameToUse = tableType == VIEW ? viewPhysicalTable.getTableName().getString() : tableName;
+ boolean sharedIndex = tableType == PTableType.INDEX && (indexType == IndexType.LOCAL || parent.getType() == PTableType.VIEW);
// For local indexes and indexes on views, pass on the the tenant id since all their meta-data rows have
// tenant ids in there.
- boolean sharedIndex = tableType == PTableType.INDEX && (indexType == IndexType.LOCAL || parent.getType() == PTableType.VIEW);
String tenantIdToUse = connection.getTenantId() != null && sharedIndex ? connection.getTenantId().getString() : null;
// When a view adds its own columns, then we need to increase the sequence number of the base table
// too since we want clients to get the latest PTable of the base table.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index ee8a268..13ad1c7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import javax.annotation.Nullable;
@@ -31,6 +32,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.index.IndexMaintainer;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryConstants;
/**
@@ -411,6 +413,19 @@ public interface PTable extends PMetaDataEntity {
private final Map<String, Integer> familyCounters = new HashMap<>();
+ /**
+ * Copy constructor
+ * @param counterToCopy
+ * @return copy of the passed counter
+ */
+ public static EncodedCQCounter copy(EncodedCQCounter counterToCopy) {
+ EncodedCQCounter cqCounter = new EncodedCQCounter();
+ for (Entry<String, Integer> e : counterToCopy.values().entrySet()) {
+ cqCounter.setValue(e.getKey(), e.getValue());
+ }
+ return cqCounter;
+ }
+
public static final EncodedCQCounter NULL_COUNTER = new EncodedCQCounter() {
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 63c1c59..ed26f17 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -1226,7 +1226,7 @@ public class PTableImpl implements PTable {
if (table.hasStorageScheme()) {
storageScheme = StorageScheme.fromSerializedValue(table.getStorageScheme().toByteArray()[0]);
}
- EncodedCQCounter encodedColumnQualifierCounter = storageScheme == StorageScheme.ENCODED_COLUMN_NAMES ? new EncodedCQCounter() : EncodedCQCounter.NULL_COUNTER;
+ EncodedCQCounter encodedColumnQualifierCounter = EncodedColumnsUtil.usesEncodedColumnNames(storageScheme) ? new EncodedCQCounter() : EncodedCQCounter.NULL_COUNTER;
if (table.getEncodedCQCountersList() != null) {
encodedColumnQualifierCounter = new EncodedCQCounter();
for (org.apache.phoenix.coprocessor.generated.PTableProtos.EncodedCQCounter cqCounterFromProto : table.getEncodedCQCountersList()) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
index 2110125..4caabbb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
@@ -32,9 +32,7 @@ import javax.annotation.concurrent.NotThreadSafe;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PTable.StorageScheme;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PInteger;
@@ -49,6 +47,8 @@ import org.apache.phoenix.schema.types.PInteger;
* implements all the optional methods of the interface too.
*
* For getting elements out o
+ *
+ * TODO: samarth think about whether we should be using this list with other storage schemes
*/
@NotThreadSafe
public class EncodedColumnQualiferCellsList implements List<Cell> {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 00ad1ae..80cb074 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -915,7 +915,7 @@ public class ScanUtil {
}
public static boolean useQualifierAsIndex(Pair<Integer, Integer> minMaxQualifiers, boolean isJoin) {
- return minMaxQualifiers != null && !isJoin;
+ return minMaxQualifiers != null;// && isJoin;
}
public static boolean setMinMaxQualifiersOnScan(PTable table) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 680dc57..ca6a471 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -63,6 +63,8 @@ import org.apache.phoenix.schema.PColumnImpl;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
+import org.apache.phoenix.schema.PTable.StorageScheme;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.TableRef;
@@ -258,7 +260,7 @@ public class CorrelatePlanTest {
PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
null, null, columns, null, null, Collections.<PTable>emptyList(),
false, Collections.<PName>emptyList(), null, null, false, false, false, null,
- null, null, true, false, 0, 0L, Boolean.FALSE, null, false, null, null);
+ null, null, true, false, 0, 0L, Boolean.FALSE, null, false, StorageScheme.NON_ENCODED_COLUMN_NAMES, EncodedCQCounter.NULL_COUNTER);
TableRef sourceTable = new TableRef(pTable);
List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
for (PColumn column : sourceTable.getTable().getColumns()) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
index bae4dd5..7fff2c4 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
@@ -57,9 +57,11 @@ import org.apache.phoenix.schema.PColumnImpl;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.PTable.StorageScheme;
import org.apache.phoenix.schema.tuple.SingleKeyValueTuple;
import org.apache.phoenix.schema.tuple.Tuple;
import org.junit.Test;
@@ -179,7 +181,7 @@ public class LiteralResultIteratorPlanTest {
PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null,
MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null,
Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false,
- false, null, null, null, true, false, 0, 0L, false, null, false, null, null);
+ false, null, null, null, true, false, 0, 0L, false, null, false, StorageScheme.NON_ENCODED_COLUMN_NAMES, EncodedCQCounter.NULL_COUNTER);
TableRef sourceTable = new TableRef(pTable);
List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
for (PColumn column : sourceTable.getTable().getColumns()) {