You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ap...@apache.org on 2015/07/08 18:50:27 UTC
[28/37] phoenix git commit: PHOENIX-2029 Queries are making two rpc
calls for getTable
PHOENIX-2029 Queries are making two rpc calls for getTable
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/52f5b046
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/52f5b046
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/52f5b046
Branch: refs/heads/4.x-HBase-1.1
Commit: 52f5b04643914f33c2d00a1157ca767a32f1adb8
Parents: 43c722c
Author: Thomas D'Silva <tw...@gmail.com>
Authored: Mon Jun 8 15:30:40 2015 -0700
Committer: Thomas D'Silva <td...@salesforce.com>
Committed: Wed Jun 17 11:16:48 2015 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 139 +++++++++++++++++++
.../apache/phoenix/compile/QueryCompiler.java | 2 +-
.../coprocessor/MetaDataEndpointImpl.java | 6 +-
.../apache/phoenix/schema/MetaDataClient.java | 26 ++--
4 files changed, 156 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/52f5b046/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
new file mode 100644
index 0000000..c657e41
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -0,0 +1,139 @@
+package org.apache.phoenix.rpc;
+
+import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
+import static org.apache.phoenix.util.TestUtil.MUTABLE_INDEX_DATA_TABLE;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isNull;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT;
+import org.apache.phoenix.end2end.Shadower;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Verifies the number of rpcs calls from {@link MetaDataClient} updateCache()
+ * for transactional and non-transactional tables.
+ */
+public class UpdateCacheIT extends BaseHBaseManagedTimeIT {
+
+ public static final int NUM_MILLIS_IN_DAY = 86400000;
+
+ @Before
+ public void setUp() throws SQLException {
+ ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE);
+ }
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException {
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "varchar" + String.valueOf(i));
+ assertEquals(rs.getString(2), "char" + String.valueOf(i));
+ assertEquals(rs.getInt(3), i);
+ assertEquals(rs.getInt(4), i);
+ assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d));
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
+ assertEquals(rs.getDate(6), date);
+ }
+
+ public static void setRowKeyColumns(PreparedStatement stmt, int i) throws SQLException {
+ // insert row
+ stmt.setString(1, "varchar" + String.valueOf(i));
+ stmt.setString(2, "char" + String.valueOf(i));
+ stmt.setInt(3, i);
+ stmt.setLong(4, i);
+ stmt.setBigDecimal(5, new BigDecimal(i*0.5d));
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * NUM_MILLIS_IN_DAY);
+ stmt.setDate(6, date);
+ }
+
+ @Test
+ public void testUpdateCache() throws Exception {
+ String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE;
+ String selectSql = "SELECT * FROM "+fullTableName;
+ // use a spyed ConnectionQueryServices so we can verify calls to getTable
+ ConnectionQueryServices connectionQueryServices = Mockito.spy(driver.getConnectionQueryServices(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)));
+ Properties props = new Properties();
+ props.putAll(PhoenixEmbeddedDriver.DEFFAULT_PROPS.asMap());
+ Connection conn = connectionQueryServices.connect(getUrl(), props);
+ try {
+ conn.setAutoCommit(false);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertFalse(rs.next());
+ reset(connectionQueryServices);
+
+ String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert three rows
+ setRowKeyColumns(stmt, 1);
+ stmt.execute();
+ setRowKeyColumns(stmt, 2);
+ stmt.execute();
+ setRowKeyColumns(stmt, 3);
+ stmt.execute();
+ conn.commit();
+ // verify only one rpc to getTable occurs after commit is called
+ verify(connectionQueryServices, times(1)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(MUTABLE_INDEX_DATA_TABLE)), anyLong(), anyLong());
+ reset(connectionQueryServices);
+
+ rs = conn.createStatement().executeQuery(selectSql);
+ validateRowKeyColumns(rs, 1);
+ validateRowKeyColumns(rs, 2);
+ validateRowKeyColumns(rs, 3);
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery(selectSql);
+ validateRowKeyColumns(rs, 1);
+ validateRowKeyColumns(rs, 2);
+ validateRowKeyColumns(rs, 3);
+ assertFalse(rs.next());
+
+ rs = conn.createStatement().executeQuery(selectSql);
+ validateRowKeyColumns(rs, 1);
+ validateRowKeyColumns(rs, 2);
+ validateRowKeyColumns(rs, 3);
+ assertFalse(rs.next());
+ conn.commit();
+ // there should be one rpc to getTable per query
+ verify(connectionQueryServices, times(3)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(INDEX_DATA_SCHEMA)), eq(PVarchar.INSTANCE.toBytes(MUTABLE_INDEX_DATA_TABLE)), anyLong(), anyLong());
+ }
+ finally {
+ conn.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/52f5b046/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
index e877e03..94ff075 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
@@ -529,7 +529,7 @@ public class QueryCompiler {
// Don't pass groupBy when building where clause expression, because we do not want to wrap these
// expressions as group by key expressions since they're pre, not post filtered.
if (innerPlan == null && !tableRef.equals(resolver.getTables().get(0))) {
- context.setResolver(FromCompiler.getResolverForQuery(select, this.statement.getConnection()));
+ context.setResolver(FromCompiler.getResolver(context.getConnection(), tableRef, select.getUdfParseNodes()));
}
Set<SubqueryParseNode> subqueries = Sets.<SubqueryParseNode> newHashSet();
Expression where = WhereCompiler.compile(context, select, viewWhere, subqueries);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/52f5b046/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 39a4956..1d578f5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -1794,10 +1794,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
if (columnToDelete.isViewReferenced()) { // Disallow deletion of column referenced in WHERE clause of view
return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION, EnvironmentEdgeManager.currentTimeMillis(), table, columnToDelete);
}
- // Look for columnToDelete in any indexes. If found as PK
- // column, get lock and drop the index. If found as covered
- // column, delete from index (do this client side?).
- // In either case, invalidate index if the column is in it
+ // Look for columnToDelete in any indexes. If found as PK column, get lock and drop the index and then invalidate it
+ // Covered columns are deleted from the index by the client
PhoenixConnection connection = table.getIndexes().isEmpty() ? null : QueryUtil.getConnection(env.getConfiguration()).unwrap(PhoenixConnection.class);
for (PTable index : table.getIndexes()) {
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/52f5b046/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index fcdb651..75678fd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -436,9 +436,9 @@ public class MetaDataClient {
// timestamp, we can handle this such that we don't ask the
// server again.
if (table != null) {
- // Ensures that table in result is set to table found in our cache.
- result.setTable(table);
if (code == MutationCode.TABLE_ALREADY_EXISTS) {
+ // Ensures that table in result is set to table found in our cache.
+ result.setTable(table);
// Although this table is up-to-date, the parent table may not be.
// In this case, we update the parent table which may in turn pull
// in indexes to add to this table.
@@ -2692,18 +2692,20 @@ public class MetaDataClient {
dropColumnMutations(table, tableColumnsToDrop, tableMetaData);
for (PTable index : table.getIndexes()) {
+ IndexMaintainer indexMaintainer = index.getIndexMaintainer(table, connection);
+ // get the columns required for the index pk
+ Set<ColumnReference> indexColumns = indexMaintainer.getIndexedColumns();
+ // get the covered columns
+ Set<ColumnReference> coveredColumns = indexMaintainer.getCoverededColumns();
List<PColumn> indexColumnsToDrop = Lists.newArrayListWithExpectedSize(columnRefs.size());
for(PColumn columnToDrop : tableColumnsToDrop) {
- String indexColumnName = IndexUtil.getIndexColumnName(columnToDrop);
- try {
- PColumn indexColumn = index.getColumn(indexColumnName);
- if (SchemaUtil.isPKColumn(indexColumn)) {
- indexesToDrop.add(new TableRef(index));
- } else {
- indexColumnsToDrop.add(indexColumn);
- columnsToDrop.add(new ColumnRef(tableRef, columnToDrop.getPosition()));
- }
- } catch (ColumnNotFoundException e) {
+ ColumnReference columnToDropRef = new ColumnReference(columnToDrop.getFamilyName().getBytes(), columnToDrop.getName().getBytes());
+ if (indexColumns.contains(columnToDropRef)) {
+ indexesToDrop.add(new TableRef(index));
+ }
+ else if (coveredColumns.contains(columnToDropRef)) {
+ String indexColumnName = IndexUtil.getIndexColumnName(columnToDrop);
+ indexColumnsToDrop.add(index.getColumn(indexColumnName));
}
}
if(!indexColumnsToDrop.isEmpty()) {