You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ra...@apache.org on 2014/10/13 10:56:57 UTC
git commit: PHOENIX-1309 Ensure Phoenix table is created for Local
index and view index tables to store guideposts against them - Addendum
(James Taylor via Ram)
Repository: phoenix
Updated Branches:
refs/heads/3.0 d42866918 -> 28d7f638f
PHOENIX-1309 Ensure Phoenix table is created for Local index and view
index tables to store guideposts against them - Addendum (James Taylor via
Ram)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/28d7f638
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/28d7f638
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/28d7f638
Branch: refs/heads/3.0
Commit: 28d7f638fca2bccfecc1d16c26efb5a3009eea6a
Parents: d428669
Author: Ramkrishna <ra...@intel.com>
Authored: Mon Oct 13 14:25:51 2014 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Mon Oct 13 14:25:51 2014 +0530
----------------------------------------------------------------------
.../end2end/TenantSpecificViewIndexIT.java | 2 +-
.../apache/phoenix/compile/PostDDLCompiler.java | 14 +--
.../phoenix/compile/StatementContext.java | 4 +
.../apache/phoenix/schema/MetaDataClient.java | 105 +++++++++++--------
.../java/org/apache/phoenix/query/BaseTest.java | 2 +-
5 files changed, 74 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28d7f638/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
index e7cdc01..8abda3b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
@@ -124,7 +124,7 @@ public class TenantSpecificViewIndexIT extends BaseTenantSpecificViewIndexIT {
}
@Test
- public void testQueryingUsingTenantSpecific() throws Exception {
+ public void testNonPaddedTenantId() throws Exception {
String tenantId1 = "org1";
String tenantId2 = "org2";
String ddl = "CREATE TABLE T (tenantId char(15) NOT NULL, pk1 varchar NOT NULL, pk2 INTEGER NOT NULL, val1 VARCHAR CONSTRAINT pk primary key (tenantId,pk1,pk2)) MULTI_TENANT = true";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28d7f638/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
index 294942f..033995e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
@@ -68,8 +68,13 @@ public class PostDDLCompiler {
private final StatementContext context; // bogus context
public PostDDLCompiler(PhoenixConnection connection) {
+ this(connection, new Scan());
+ }
+
+ public PostDDLCompiler(PhoenixConnection connection, Scan scan) {
this.connection = connection;
- this.context = new StatementContext(new PhoenixStatement(connection));
+ this.context = new StatementContext(new PhoenixStatement(connection), scan);
+ scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
public MutationPlan compile(final List<TableRef> tableRefs, final byte[] emptyCF, final byte[] projectCF, final List<PColumn> deleteList,
@@ -101,19 +106,16 @@ public class PostDDLCompiler {
try {
connection.setAutoCommit(true);
SQLException sqlE = null;
- if (deleteList == null && emptyCF == null) {
- return new MutationState(0, connection);
- }
/*
* Handles:
* 1) deletion of all rows for a DROP TABLE and subsequently deletion of all rows for a DROP INDEX;
* 2) deletion of all column values for a ALTER TABLE DROP COLUMN
* 3) updating the necessary rows to have an empty KV
+ * 4) updating table stats
*/
long totalMutationCount = 0;
for (final TableRef tableRef : tableRefs) {
- Scan scan = new Scan();
- scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
+ Scan scan = ScanUtil.newScan(context.getScan());
SelectStatement select = SelectStatement.COUNT_ONE;
// We need to use this tableRef
ColumnResolver resolver = new ColumnResolver() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28d7f638/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 887ca3e..b513ac1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -75,6 +75,10 @@ public class StatementContext {
private Map<SelectStatement, Object> subqueryResults;
public StatementContext(PhoenixStatement statement) {
+ this(statement, new Scan());
+ }
+
+ public StatementContext(PhoenixStatement statement, Scan scan) {
this(statement, FromCompiler.EMPTY_TABLE_RESOLVER, new Scan(), new SequenceManager(statement));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28d7f638/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index b0549d2..a9a59d3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -84,13 +84,11 @@ import java.util.Set;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -99,7 +97,6 @@ import org.apache.phoenix.compile.FromCompiler;
import org.apache.phoenix.compile.MutationPlan;
import org.apache.phoenix.compile.PostDDLCompiler;
import org.apache.phoenix.compile.PostIndexDDLCompiler;
-import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
@@ -109,7 +106,6 @@ import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.parse.AddColumnStatement;
import org.apache.phoenix.parse.AlterIndexStatement;
import org.apache.phoenix.parse.ColumnDef;
@@ -484,33 +480,55 @@ public class MetaDataClient {
public MutationState updateStatistics(UpdateStatisticsStatement updateStatisticsStmt)
throws SQLException {
// Check before updating the stats if we have reached the configured time to reupdate the stats once again
- ReadOnlyProps props = connection.getQueryServices().getProps();
- final long msMinBetweenUpdates = props
- .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
- props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2);
ColumnResolver resolver = FromCompiler.getResolver(updateStatisticsStmt, connection);
PTable table = resolver.getTables().get(0).getTable();
- List<PTable> indexes = table.getIndexes();
- List<PTable> tables = Lists.newArrayListWithExpectedSize(1 + indexes.size());
+ long rowCount = 0;
if (updateStatisticsStmt.updateColumns()) {
- tables.add(table);
+ rowCount += updateStatisticsInternal(table.getPhysicalName(), table);
}
if (updateStatisticsStmt.updateIndex()) {
- tables.addAll(indexes);
- }
- for(PTable pTable : tables) {
- updateStatisticsInternal(msMinBetweenUpdates, pTable);
+ // TODO: If our table is a VIEW with multiple indexes or a TABLE with local indexes,
+ // we may be doing more work that we have to here. We should union the scan ranges
+ // across all indexes in that case so that we don't re-calculate the same stats
+ // multiple times.
+ for (PTable index : table.getIndexes()) {
+ rowCount += updateStatisticsInternal(index.getPhysicalName(), index);
+ }
+ // If analyzing the indexes of a multi-tenant table or a table with view indexes
+ // then analyze all of those indexes too.
+ if (table.getType() != PTableType.VIEW &&
+ (table.isMultiTenant())
+ || (MetaDataUtil.hasViewIndexTable(connection, table.getName()))){
+
+ String viewIndexTableName = MetaDataUtil.getViewIndexTableName(table.getTableName().getString());
+ String viewIndexSchemaName = MetaDataUtil.getViewIndexSchemaName(table.getSchemaName().getString());
+ final PName viewIndexPhysicalName = PNameFactory.newName(SchemaUtil.getTableName(viewIndexSchemaName, viewIndexTableName));
+ PTable indexLogicalTable = new DelegateTable(table) {
+ @Override
+ public PName getPhysicalName() {
+ return viewIndexPhysicalName;
+ }
+ @Override
+ public PTableStats getTableStats() {
+ return PTableStats.EMPTY_STATS;
+ }
+ };
+ rowCount += updateStatisticsInternal(viewIndexPhysicalName, indexLogicalTable);
+ }
}
- return new MutationState(1, connection);
+ return new MutationState((int)rowCount, connection);
}
- private MutationState updateStatisticsInternal(long msMinBetweenUpdates, PTable table) throws SQLException {
- PName physicalName = table.getPhysicalName();
+ private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException {
+ ReadOnlyProps props = connection.getQueryServices().getProps();
+ final long msMinBetweenUpdates = props
+ .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
+ props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2);
byte[] tenantIdBytes = ByteUtil.EMPTY_BYTE_ARRAY;
Long scn = connection.getSCN();
// Always invalidate the cache
- long clientTS = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn;
+ long clientTimeStamp = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn;
String query = "SELECT CURRENT_DATE() - " + LAST_STATS_UPDATE_TIME + " FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME
+ " WHERE " + PHYSICAL_NAME + "='" + physicalName.getString() + "' AND " + COLUMN_FAMILY
+ " IS NULL AND " + REGION_NAME + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL";
@@ -519,32 +537,29 @@ public class MetaDataClient {
if (rs.next()) {
msSinceLastUpdate = rs.getLong(1);
}
- if (msSinceLastUpdate >= msMinBetweenUpdates) {
- // Here create the select query.
- String countQuery = "SELECT /*+ NO_CACHE NO_INDEX */ count(*) FROM " + table.getName().getString();
- PhoenixStatement statement = (PhoenixStatement) connection.createStatement();
- QueryPlan plan = statement.compileQuery(countQuery);
- Scan scan = plan.getContext().getScan();
- // Add all CF in the table
- scan.getFamilyMap().clear();
- for (PColumnFamily family : table.getColumnFamilies()) {
- scan.addFamily(family.getName().getBytes());
- }
- scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
- KeyValue kv = plan.iterator().next().getValue(0);
- ImmutableBytesWritable tempPtr = plan.getContext().getTempPtr();
- tempPtr.set(kv.getValue());
- // A single Cell will be returned with the count(*) - we decode that here
- long rowCount = PDataType.LONG.getCodec().decodeLong(tempPtr, SortOrder.getDefault());
- // We need to update the stats table so that client will pull the new one with
- // the updated stats.
- connection.getQueryServices().incrementTableTimeStamp(tenantIdBytes,
- Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
- Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTS);
- return new MutationState(0, connection, rowCount);
- } else {
- return new MutationState(0, connection);
+ if (msSinceLastUpdate < msMinBetweenUpdates) {
+ return 0;
}
+
+ /*
+ * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through, since it may not represent a "real"
+ * table in the case of the view indexes of a base table.
+ */
+ PostDDLCompiler compiler = new PostDDLCompiler(connection);
+ TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false);
+ MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
+ Scan scan = plan.getContext().getScan();
+ scan.setCacheBlocks(false);
+ scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
+ MutationState mutationState = plan.execute();
+ long rowCount = mutationState.getUpdateCount();
+
+ // We need to update the stats table so that client will pull the new one with
+ // the updated stats.
+ connection.getQueryServices().incrementTableTimeStamp(tenantIdBytes,
+ Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
+ Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
+ return rowCount;
}
private MutationState buildIndexAtTimeStamp(PTable index, NamedTableNode dataTableNode) throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/28d7f638/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index fa07bfb..1e3f004 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -551,7 +551,7 @@ public abstract class BaseTest {
@Override
public void run() {
try {
- utility.shutdownMiniCluster();
+ if (utility != null) utility.shutdownMiniCluster();
} catch (Exception e) {
logger.warn("Exception caught when shutting down mini cluster", e);
}