You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2016/11/03 23:47:11 UTC
[2/2] phoenix git commit: PHOENIX-3421 Column name lookups fail when
on an indexed table
PHOENIX-3421 Column name lookups fail when on an indexed table
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/49914c2e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/49914c2e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/49914c2e
Branch: refs/heads/master
Commit: 49914c2e8062bf6b175dc987fac97b3f0e659dbe
Parents: a6c9024
Author: James Taylor <ja...@apache.org>
Authored: Thu Nov 3 16:21:28 2016 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Thu Nov 3 16:47:01 2016 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/QueryMoreIT.java | 4 +-
.../org/apache/phoenix/util/PhoenixRuntime.java | 132 ++++++++++++++++++-
.../phoenix/util/PhoenixEncodeDecodeTest.java | 4 +-
.../apache/phoenix/util/PhoenixRuntimeTest.java | 8 +-
4 files changed, 137 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
index b9162de..2b27f00 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
@@ -275,7 +275,7 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
values[i] = rs.getObject(i + 1);
}
conn = getTenantSpecificConnection(tenantId);
- pkIds.add(Base64.encodeBytes(PhoenixRuntime.encodeValues(conn, tableOrViewName.toUpperCase(), values, columns)));
+ pkIds.add(Base64.encodeBytes(PhoenixRuntime.encodeColumnValues(conn, tableOrViewName.toUpperCase(), values, columns)));
}
return pkIds.toArray(new String[pkIds.size()]);
}
@@ -293,7 +293,7 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
PreparedStatement stmt = conn.prepareStatement(query);
int bindCounter = 1;
for (int i = 0; i < cursorIds.length; i++) {
- Object[] pkParts = PhoenixRuntime.decodeValues(conn, tableName.toUpperCase(), Base64.decode(cursorIds[i]), columns);
+ Object[] pkParts = PhoenixRuntime.decodeColumnValues(conn, tableName.toUpperCase(), Base64.decode(cursorIds[i]), columns);
for (int j = 0; j < pkParts.length; j++) {
stmt.setObject(bindCounter++, pkParts[j]);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
index 5dd4592..dbac76f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
@@ -816,7 +816,7 @@ public class PhoenixRuntime {
public static List<Pair<String, String>> getPkColsForSql(Connection conn, QueryPlan plan) throws SQLException {
checkNotNull(plan);
checkNotNull(conn);
- List<PColumn> pkColumns = getPkColumns(plan.getTableRef().getTable(), conn, true);
+ List<PColumn> pkColumns = getPkColumns(plan.getTableRef().getTable(), conn);
List<Pair<String, String>> columns = Lists.newArrayListWithExpectedSize(pkColumns.size());
String columnName;
String familyName;
@@ -921,6 +921,7 @@ public class PhoenixRuntime {
return sqlTypeName;
}
+ @Deprecated
private static List<PColumn> getPkColumns(PTable ptable, Connection conn, boolean forDataTable) throws SQLException {
PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class);
List<PColumn> pkColumns = ptable.getPKColumns();
@@ -943,6 +944,28 @@ public class PhoenixRuntime {
return pkColumns;
}
+ private static List<PColumn> getPkColumns(PTable ptable, Connection conn) throws SQLException {
+ PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class);
+ List<PColumn> pkColumns = ptable.getPKColumns();
+
+ // Skip the salting column and the view index id column if present.
+ // Skip the tenant id column too if the connection is tenant specific and the table used by the query plan is multi-tenant
+ int offset = (ptable.getBucketNum() == null ? 0 : 1) + (ptable.isMultiTenant() && pConn.getTenantId() != null ? 1 : 0) + (ptable.getViewIndexId() == null ? 0 : 1);
+
+ // get a sublist of pkColumns by skipping the offset columns.
+ pkColumns = pkColumns.subList(offset, pkColumns.size());
+
+ if (ptable.getType() == PTableType.INDEX) {
+ // index tables have the same schema name as their parent/data tables.
+ String fullDataTableName = ptable.getParentName().getString();
+
+ // Get the corresponding columns of the data table.
+ List<PColumn> dataColumns = IndexUtil.getDataColumns(fullDataTableName, pkColumns, pConn);
+ pkColumns = dataColumns;
+ }
+ return pkColumns;
+ }
+
/**
*
* @param conn connection that was used for reading/generating value.
@@ -955,6 +978,7 @@ public class PhoenixRuntime {
* @throws SQLException
* @see {@link #decodeValues(Connection, String, byte[], List)}
*/
+ @Deprecated
public static byte[] encodeValues(Connection conn, String fullTableName, Object[] values, List<Pair<String, String>> columns) throws SQLException {
PTable table = getTable(conn, fullTableName);
List<PColumn> pColumns = getPColumns(table, columns);
@@ -978,7 +1002,7 @@ public class PhoenixRuntime {
*
* @param conn connection that was used for reading/generating value.
* @param fullTableName fully qualified table name
- * @param value byte value of the columns concatenated as a single byte array. @see {@link #encodeValues(Connection, String, Object[], List)}
+ * @param value byte value of the columns concatenated as a single byte array. @see {@link #encodeColumnValues(Connection, String, Object[], List)}
* @param columns list of column names for the columns that have their respective values
* present in the byte array. The column names should be in the same order as their values are in the byte array.
* The column name includes both family name, if present, and column name.
@@ -986,6 +1010,7 @@ public class PhoenixRuntime {
* @throws SQLException
*
*/
+ @Deprecated
public static Object[] decodeValues(Connection conn, String fullTableName, byte[] value, List<Pair<String, String>> columns) throws SQLException {
PTable table = getTable(conn, fullTableName);
KeyValueSchema kvSchema = buildKeyValueSchema(getPColumns(table, columns));
@@ -1007,6 +1032,70 @@ public class PhoenixRuntime {
return values.toArray();
}
+ /**
+ *
+ * @param conn connection that was used for reading/generating value.
+ * @param fullTableName fully qualified table name
+ * @param values values of the columns
+ * @param columns list of pair of column that includes column family as first part and column name as the second part.
+ * Column family is optional and hence nullable. Columns in the list have to be in the same order as the order of occurence
+ * of their values in the object array.
+ * @return values encoded in a byte array
+ * @throws SQLException
+ * @see {@link #decodeValues(Connection, String, byte[], List)}
+ */
+ public static byte[] encodeColumnValues(Connection conn, String fullTableName, Object[] values, List<Pair<String, String>> columns) throws SQLException {
+ PTable table = getTable(conn, fullTableName);
+ List<PColumn> pColumns = getColumns(table, columns);
+ List<Expression> expressions = new ArrayList<Expression>(pColumns.size());
+ int i = 0;
+ for (PColumn col : pColumns) {
+ Object value = values[i];
+ // for purposes of encoding, sort order of the columns doesn't matter.
+ Expression expr = LiteralExpression.newConstant(value, col.getDataType(), col.getMaxLength(), col.getScale());
+ expressions.add(expr);
+ i++;
+ }
+ KeyValueSchema kvSchema = buildKeyValueSchema(pColumns);
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ ValueBitSet valueSet = ValueBitSet.newInstance(kvSchema);
+ return kvSchema.toBytes(expressions.toArray(new Expression[0]), valueSet, ptr);
+ }
+
+
+ /**
+ *
+ * @param conn connection that was used for reading/generating value.
+ * @param fullTableName fully qualified table name
+ * @param value byte value of the columns concatenated as a single byte array. @see {@link #encodeColumnValues(Connection, String, Object[], List)}
+ * @param columns list of column names for the columns that have their respective values
+ * present in the byte array. The column names should be in the same order as their values are in the byte array.
+ * The column name includes both family name, if present, and column name.
+ * @return decoded values for each column
+ * @throws SQLException
+ *
+ */
+ public static Object[] decodeColumnValues(Connection conn, String fullTableName, byte[] value, List<Pair<String, String>> columns) throws SQLException {
+ PTable table = getTable(conn, fullTableName);
+ KeyValueSchema kvSchema = buildKeyValueSchema(getColumns(table, columns));
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable(value);
+ ValueBitSet valueSet = ValueBitSet.newInstance(kvSchema);
+ valueSet.clear();
+ valueSet.or(ptr);
+ int maxOffset = ptr.getOffset() + ptr.getLength();
+ Boolean hasValue;
+ kvSchema.iterator(ptr);
+ int i = 0;
+ List<Object> values = new ArrayList<Object>();
+ while(hasValue = kvSchema.next(ptr, i, maxOffset, valueSet) != null) {
+ if(hasValue) {
+ values.add(kvSchema.getField(i).getDataType().toObject(ptr));
+ }
+ i++;
+ }
+ return values.toArray();
+ }
+
private static KeyValueSchema buildKeyValueSchema(List<PColumn> columns) {
KeyValueSchemaBuilder builder = new KeyValueSchemaBuilder(getMinNullableIndex(columns));
for (PColumn col : columns) {
@@ -1026,13 +1115,14 @@ public class PhoenixRuntime {
return minNullableIndex;
}
- /**
+ /**
* @param table table to get the {@code PColumn} for
* @param columns list of pair of column that includes column family as first part and column name as the second part.
* Column family is optional and hence nullable.
* @return list of {@code PColumn} for fullyQualifiedColumnNames
* @throws SQLException
*/
+ @Deprecated
private static List<PColumn> getPColumns(PTable table, List<Pair<String, String>> columns) throws SQLException {
List<PColumn> pColumns = new ArrayList<PColumn>(columns.size());
for (Pair<String, String> column : columns) {
@@ -1041,6 +1131,7 @@ public class PhoenixRuntime {
return pColumns;
}
+ @Deprecated
private static PColumn getPColumn(PTable table, @Nullable String familyName, String columnName) throws SQLException {
if (table==null) {
throw new SQLException("Table must not be null.");
@@ -1051,6 +1142,41 @@ public class PhoenixRuntime {
// normalize and remove quotes from family and column names before looking up.
familyName = SchemaUtil.normalizeIdentifier(familyName);
columnName = SchemaUtil.normalizeIdentifier(columnName);
+ PColumn pColumn = null;
+ if (familyName != null) {
+ PColumnFamily family = table.getColumnFamily(familyName);
+ pColumn = family.getColumn(columnName);
+ } else {
+ pColumn = table.getColumn(columnName);
+ }
+ return pColumn;
+ }
+
+ /**
+ * @param table table to get the {@code PColumn} for
+ * @param columns list of pair of column that includes column family as first part and column name as the second part.
+ * Column family is optional and hence nullable.
+ * @return list of {@code PColumn} for fullyQualifiedColumnNames
+ * @throws SQLException
+ */
+ private static List<PColumn> getColumns(PTable table, List<Pair<String, String>> columns) throws SQLException {
+ List<PColumn> pColumns = new ArrayList<PColumn>(columns.size());
+ for (Pair<String, String> column : columns) {
+ pColumns.add(getColumn(table, column.getFirst(), column.getSecond()));
+ }
+ return pColumns;
+ }
+
+ private static PColumn getColumn(PTable table, @Nullable String familyName, String columnName) throws SQLException {
+ if (table==null) {
+ throw new SQLException("Table must not be null.");
+ }
+ if (columnName==null) {
+ throw new SQLException("columnName must not be null.");
+ }
+ // normalize and remove quotes from family and column names before looking up.
+ familyName = SchemaUtil.normalizeIdentifier(familyName);
+ columnName = SchemaUtil.normalizeIdentifier(columnName);
// Column names are always for the data table, so we must translate them if
// we're dealing with an index table.
if (table.getType() == PTableType.INDEX) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java
index 85338c4..56b3f45 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixEncodeDecodeTest.java
@@ -62,8 +62,8 @@ public class PhoenixEncodeDecodeTest extends BaseConnectionlessQueryTest {
Date d = nullFixedWidth ? null : new Date(100);
String s = nullVariableWidth ? null : "foo";
Object[] values = new Object[] {"def", "eid", d, s, s};
- byte[] bytes = PhoenixRuntime.encodeValues(conn, "T", values, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1")));
- Object[] decodedValues = PhoenixRuntime.decodeValues(conn, "T", bytes, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1")));
+ byte[] bytes = PhoenixRuntime.encodeColumnValues(conn, "T", values, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1")));
+ Object[] decodedValues = PhoenixRuntime.decodeColumnValues(conn, "T", bytes, Lists.newArrayList(new Pair<String, String>(null, "pk1"), new Pair<String, String>(null, "pk2"), new Pair<String, String>("cf1", "v1"), new Pair<String, String>("cf2", "v2"), new Pair<String, String>("cf2", "v1")));
assertEquals(Lists.newArrayList("def", "eid", d, s, s), Arrays.asList(decodedValues));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/49914c2e/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
index 783ab17..430c20b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
@@ -131,8 +131,8 @@ public class PhoenixRuntimeTest extends BaseConnectionlessQueryTest {
List<Pair<String,String>> pkColumns = PhoenixRuntime.getPkColsForSql(conn, plan);
String fullTableName = plan.getTableRef().getTable().getName().getString();
assertEquals("I", fullTableName);
- byte[] encodedValues = PhoenixRuntime.encodeValues(conn, fullTableName, values, pkColumns);
- Object[] decodedValues = PhoenixRuntime.decodeValues(conn, fullTableName, encodedValues, pkColumns);
+ byte[] encodedValues = PhoenixRuntime.encodeColumnValues(conn, fullTableName, values, pkColumns);
+ Object[] decodedValues = PhoenixRuntime.decodeColumnValues(conn, fullTableName, encodedValues, pkColumns);
assertArrayEquals(values, decodedValues);
plan = conn.createStatement().unwrap(PhoenixStatement.class).optimizeQuery("SELECT /*+ NO_INDEX */ ENTITY_HISTORY_ID FROM T");
@@ -140,8 +140,8 @@ public class PhoenixRuntimeTest extends BaseConnectionlessQueryTest {
values = new Object[] {tenantId, parentId, createdDate, ehId};
fullTableName = plan.getTableRef().getTable().getName().getString();
assertEquals("T", fullTableName);
- encodedValues = PhoenixRuntime.encodeValues(conn, fullTableName, values, pkColumns);
- decodedValues = PhoenixRuntime.decodeValues(conn, fullTableName, encodedValues, pkColumns);
+ encodedValues = PhoenixRuntime.encodeColumnValues(conn, fullTableName, values, pkColumns);
+ decodedValues = PhoenixRuntime.decodeColumnValues(conn, fullTableName, encodedValues, pkColumns);
assertArrayEquals(values, decodedValues);
}