You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/10/11 11:52:29 UTC
[3/3] git commit: PHOENIX-1309 Ensure Phoenix table is created for
Local index and view index tables to store guideposts against them
PHOENIX-1309 Ensure Phoenix table is created for Local index and view index tables to store guideposts against them
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c64cef6d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c64cef6d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c64cef6d
Branch: refs/heads/master
Commit: c64cef6d80eb7af6f26fdc82db9c9be1e14e8184
Parents: 43afafb
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Oct 10 23:25:30 2014 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Sat Oct 11 02:56:56 2014 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/BaseViewIT.java | 6 +-
.../end2end/TenantSpecificViewIndexIT.java | 2 +-
.../apache/phoenix/compile/PostDDLCompiler.java | 14 +-
.../phoenix/compile/StatementContext.java | 4 +
.../apache/phoenix/schema/DelegateTable.java | 228 +++++++++++++++++++
.../apache/phoenix/schema/MetaDataClient.java | 106 +++++----
.../java/org/apache/phoenix/query/BaseTest.java | 2 +-
7 files changed, 308 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c64cef6d/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
index 13e743f..2fe7fd1 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
@@ -150,9 +150,9 @@ public abstract class BaseViewIT extends BaseOwnClusterHBaseManagedTimeIT {
assertEquals(saltBuckets == null ? 1 : 3, splits.size());
// analyze table should analyze all view data
-// analyzeTable(conn, "t");
-// splits = getAllSplits(conn, "i2");
-// assertEquals(saltBuckets == null ? 6 : 8, splits.size());
+ analyzeTable(conn, "t");
+ splits = getAllSplits(conn, "i2");
+ assertEquals(saltBuckets == null ? 6 : 8, splits.size());
query = "SELECT k1, k2, s FROM v WHERE s = 'foo'";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c64cef6d/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
index 4ccdef4..094222b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
@@ -151,7 +151,7 @@ public class TenantSpecificViewIndexIT extends BaseTenantSpecificViewIndexIT {
}
@Test
- public void testQueryingUsingTenantSpecific() throws Exception {
+ public void testNonPaddedTenantId() throws Exception {
String tenantId1 = "org1";
String tenantId2 = "org2";
String ddl = "CREATE TABLE T (tenantId char(15) NOT NULL, pk1 varchar NOT NULL, pk2 INTEGER NOT NULL, val1 VARCHAR CONSTRAINT pk primary key (tenantId,pk1,pk2)) MULTI_TENANT = true";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c64cef6d/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
index 294942f..033995e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
@@ -68,8 +68,13 @@ public class PostDDLCompiler {
private final StatementContext context; // bogus context
public PostDDLCompiler(PhoenixConnection connection) {
+ this(connection, new Scan());
+ }
+
+ public PostDDLCompiler(PhoenixConnection connection, Scan scan) {
this.connection = connection;
- this.context = new StatementContext(new PhoenixStatement(connection));
+ this.context = new StatementContext(new PhoenixStatement(connection), scan);
+ scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
public MutationPlan compile(final List<TableRef> tableRefs, final byte[] emptyCF, final byte[] projectCF, final List<PColumn> deleteList,
@@ -101,19 +106,16 @@ public class PostDDLCompiler {
try {
connection.setAutoCommit(true);
SQLException sqlE = null;
- if (deleteList == null && emptyCF == null) {
- return new MutationState(0, connection);
- }
/*
* Handles:
* 1) deletion of all rows for a DROP TABLE and subsequently deletion of all rows for a DROP INDEX;
* 2) deletion of all column values for a ALTER TABLE DROP COLUMN
* 3) updating the necessary rows to have an empty KV
+ * 4) updating table stats
*/
long totalMutationCount = 0;
for (final TableRef tableRef : tableRefs) {
- Scan scan = new Scan();
- scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
+ Scan scan = ScanUtil.newScan(context.getScan());
SelectStatement select = SelectStatement.COUNT_ONE;
// We need to use this tableRef
ColumnResolver resolver = new ColumnResolver() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c64cef6d/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 242fc45..b0ba6f0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -79,6 +79,10 @@ public class StatementContext {
private Map<SelectStatement, Object> subqueryResults;
public StatementContext(PhoenixStatement statement) {
+ this(statement, new Scan());
+ }
+
+ public StatementContext(PhoenixStatement statement, Scan scan) {
this(statement, FromCompiler.EMPTY_TABLE_RESOLVER, new Scan(), new SequenceManager(statement));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c64cef6d/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
new file mode 100644
index 0000000..f4bf1cc
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
+import org.apache.phoenix.index.IndexMaintainer;
+import org.apache.phoenix.schema.stats.PTableStats;
+
+public class DelegateTable implements PTable {
+ @Override
+ public long getTimeStamp() {
+ return delegate.getTimeStamp();
+ }
+
+ @Override
+ public long getSequenceNumber() {
+ return delegate.getSequenceNumber();
+ }
+
+ @Override
+ public PName getName() {
+ return delegate.getName();
+ }
+
+ @Override
+ public PName getSchemaName() {
+ return delegate.getSchemaName();
+ }
+
+ @Override
+ public PName getTableName() {
+ return delegate.getTableName();
+ }
+
+ @Override
+ public PName getTenantId() {
+ return delegate.getTenantId();
+ }
+
+ @Override
+ public PTableType getType() {
+ return delegate.getType();
+ }
+
+ @Override
+ public PName getPKName() {
+ return delegate.getPKName();
+ }
+
+ @Override
+ public List<PColumn> getPKColumns() {
+ return delegate.getPKColumns();
+ }
+
+ @Override
+ public List<PColumn> getColumns() {
+ return delegate.getColumns();
+ }
+
+ @Override
+ public List<PColumnFamily> getColumnFamilies() {
+ return delegate.getColumnFamilies();
+ }
+
+ @Override
+ public PColumnFamily getColumnFamily(byte[] family) throws ColumnFamilyNotFoundException {
+ return delegate.getColumnFamily(family);
+ }
+
+ @Override
+ public PColumnFamily getColumnFamily(String family) throws ColumnFamilyNotFoundException {
+ return delegate.getColumnFamily(family);
+ }
+
+ @Override
+ public PColumn getColumn(String name) throws ColumnNotFoundException, AmbiguousColumnException {
+ return delegate.getColumn(name);
+ }
+
+ @Override
+ public PColumn getPKColumn(String name) throws ColumnNotFoundException {
+ return delegate.getPKColumn(name);
+ }
+
+ @Override
+ public PRow newRow(KeyValueBuilder builder, long ts, ImmutableBytesWritable key, byte[]... values) {
+ return delegate.newRow(builder, ts, key, values);
+ }
+
+ @Override
+ public PRow newRow(KeyValueBuilder builder, ImmutableBytesWritable key, byte[]... values) {
+ return delegate.newRow(builder, key, values);
+ }
+
+ @Override
+ public int newKey(ImmutableBytesWritable key, byte[][] values) {
+ return delegate.newKey(key, values);
+ }
+
+ @Override
+ public RowKeySchema getRowKeySchema() {
+ return delegate.getRowKeySchema();
+ }
+
+ @Override
+ public Integer getBucketNum() {
+ return delegate.getBucketNum();
+ }
+
+ @Override
+ public List<PTable> getIndexes() {
+ return delegate.getIndexes();
+ }
+
+ @Override
+ public PIndexState getIndexState() {
+ return delegate.getIndexState();
+ }
+
+ @Override
+ public PName getParentName() {
+ return delegate.getParentName();
+ }
+
+ @Override
+ public PName getParentTableName() {
+ return delegate.getParentTableName();
+ }
+
+ @Override
+ public List<PName> getPhysicalNames() {
+ return delegate.getPhysicalNames();
+ }
+
+ @Override
+ public PName getPhysicalName() {
+ return delegate.getPhysicalName();
+ }
+
+ @Override
+ public boolean isImmutableRows() {
+ return delegate.isImmutableRows();
+ }
+
+ @Override
+ public void getIndexMaintainers(ImmutableBytesWritable ptr) {
+ delegate.getIndexMaintainers(ptr);
+ }
+
+ @Override
+ public IndexMaintainer getIndexMaintainer(PTable dataTable) {
+ return delegate.getIndexMaintainer(dataTable);
+ }
+
+ @Override
+ public PName getDefaultFamilyName() {
+ return delegate.getDefaultFamilyName();
+ }
+
+ @Override
+ public boolean isWALDisabled() {
+ return delegate.isWALDisabled();
+ }
+
+ @Override
+ public boolean isMultiTenant() {
+ return delegate.isMultiTenant();
+ }
+
+ @Override
+ public ViewType getViewType() {
+ return delegate.getViewType();
+ }
+
+ @Override
+ public String getViewStatement() {
+ return delegate.getViewStatement();
+ }
+
+ @Override
+ public Short getViewIndexId() {
+ return delegate.getViewIndexId();
+ }
+
+ @Override
+ public PTableKey getKey() {
+ return delegate.getKey();
+ }
+
+ @Override
+ public int getEstimatedSize() {
+ return delegate.getEstimatedSize();
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ return delegate.getIndexType();
+ }
+
+ @Override
+ public PTableStats getTableStats() {
+ return delegate.getTableStats();
+ }
+
+ private final PTable delegate;
+
+ public DelegateTable(PTable delegate) {
+ this.delegate = delegate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c64cef6d/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 9b949b7..8e63cf3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -490,33 +490,58 @@ public class MetaDataClient {
public MutationState updateStatistics(UpdateStatisticsStatement updateStatisticsStmt)
throws SQLException {
// Check before updating the stats if we have reached the configured time to reupdate the stats once again
- ReadOnlyProps props = connection.getQueryServices().getProps();
- final long msMinBetweenUpdates = props
- .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
- props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2);
ColumnResolver resolver = FromCompiler.getResolver(updateStatisticsStmt, connection);
PTable table = resolver.getTables().get(0).getTable();
- List<PTable> indexes = table.getIndexes();
- List<PTable> tables = Lists.newArrayListWithExpectedSize(1 + indexes.size());
+ long rowCount = 0;
if (updateStatisticsStmt.updateColumns()) {
- tables.add(table);
+ rowCount += updateStatisticsInternal(table.getPhysicalName(), table);
}
if (updateStatisticsStmt.updateIndex()) {
- tables.addAll(indexes);
- }
- for(PTable pTable : tables) {
- updateStatisticsInternal(msMinBetweenUpdates, pTable);
+ // TODO: If our table is a VIEW with multiple indexes or a TABLE with local indexes,
+ // we may be doing more work that we have to here. We should union the scan ranges
+ // across all indexes in that case so that we don't re-calculate the same stats
+ // multiple times.
+ for (PTable index : table.getIndexes()) {
+ rowCount += updateStatisticsInternal(index.getPhysicalName(), index);
+ }
+ // If analyzing the indexes of a multi-tenant table or a table with view indexes
+ // then analyze all of those indexes too.
+ boolean isLocalIndex = false;
+ boolean isViewIndex = false;
+ if (table.getType() != PTableType.VIEW &&
+ (table.isMultiTenant())
+ || (isViewIndex = MetaDataUtil.hasViewIndexTable(connection, table.getName()))
+ || (isLocalIndex = MetaDataUtil.hasLocalIndexTable(connection, table.getName()))) {
+
+ String viewIndexTableName = isLocalIndex ? MetaDataUtil.getLocalIndexTableName(table.getTableName().getString()) : MetaDataUtil.getViewIndexTableName(table.getTableName().getString());
+ String viewIndexSchemaName = isLocalIndex ? MetaDataUtil.getLocalIndexSchemaName(table.getSchemaName().getString()) : MetaDataUtil.getViewIndexSchemaName(table.getSchemaName().getString());
+ final PName viewIndexPhysicalName = PNameFactory.newName(SchemaUtil.getTableName(viewIndexSchemaName, viewIndexTableName));
+ PTable indexLogicalTable = new DelegateTable(table) {
+ @Override
+ public PName getPhysicalName() {
+ return viewIndexPhysicalName;
+ }
+ @Override
+ public PTableStats getTableStats() {
+ return PTableStats.EMPTY_STATS;
+ }
+ };
+ rowCount += updateStatisticsInternal(viewIndexPhysicalName, indexLogicalTable);
+ }
}
- return new MutationState(1, connection);
+ return new MutationState((int)rowCount, connection);
}
- private MutationState updateStatisticsInternal(long msMinBetweenUpdates, PTable table) throws SQLException {
- PName physicalName = table.getPhysicalName();
+ private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException {
+ ReadOnlyProps props = connection.getQueryServices().getProps();
+ final long msMinBetweenUpdates = props
+ .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
+ props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2);
byte[] tenantIdBytes = ByteUtil.EMPTY_BYTE_ARRAY;
Long scn = connection.getSCN();
// Always invalidate the cache
- long clientTS = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn;
+ long clientTimeStamp = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn;
String query = "SELECT CURRENT_DATE() - " + LAST_STATS_UPDATE_TIME + " FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME
+ " WHERE " + PHYSICAL_NAME + "='" + physicalName.getString() + "' AND " + COLUMN_FAMILY
+ " IS NULL AND " + REGION_NAME + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL";
@@ -525,32 +550,29 @@ public class MetaDataClient {
if (rs.next()) {
msSinceLastUpdate = rs.getLong(1);
}
- if (msSinceLastUpdate >= msMinBetweenUpdates) {
- // Here create the select query.
- String countQuery = "SELECT /*+ NO_CACHE NO_INDEX */ count(*) FROM " + table.getName().getString();
- PhoenixStatement statement = (PhoenixStatement) connection.createStatement();
- QueryPlan plan = statement.compileQuery(countQuery);
- Scan scan = plan.getContext().getScan();
- // Add all CF in the table
- scan.getFamilyMap().clear();
- for (PColumnFamily family : table.getColumnFamilies()) {
- scan.addFamily(family.getName().getBytes());
- }
- scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
- Cell kv = plan.iterator().next().getValue(0);
- ImmutableBytesWritable tempPtr = plan.getContext().getTempPtr();
- tempPtr.set(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
- // A single Cell will be returned with the count(*) - we decode that here
- long rowCount = PDataType.LONG.getCodec().decodeLong(tempPtr, SortOrder.getDefault());
- // We need to update the stats table so that client will pull the new one with
- // the updated stats.
- connection.getQueryServices().incrementTableTimeStamp(tenantIdBytes,
- Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
- Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTS);
- return new MutationState(0, connection, rowCount);
- } else {
- return new MutationState(0, connection);
+ if (msSinceLastUpdate < msMinBetweenUpdates) {
+ return 0;
}
+
+ /*
+ * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through,
+ * since it may not represent a "real" table in the case of the view indexes of a base table.
+ */
+ PostDDLCompiler compiler = new PostDDLCompiler(connection);
+ TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false);
+ MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
+ Scan scan = plan.getContext().getScan();
+ scan.setCacheBlocks(false);
+ scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
+ MutationState mutationState = plan.execute();
+ long rowCount = mutationState.getUpdateCount();
+
+ // We need to update the stats table so that client will pull the new one with
+ // the updated stats.
+ connection.getQueryServices().incrementTableTimeStamp(tenantIdBytes,
+ Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
+ Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
+ return rowCount;
}
private MutationState buildIndexAtTimeStamp(PTable index, NamedTableNode dataTableNode) throws SQLException {
@@ -970,7 +992,6 @@ public class MetaDataClient {
String parentTableName = null;
PName tenantId = connection.getTenantId();
String tenantIdStr = tenantId == null ? null : connection.getTenantId().getString();
- boolean isParentImmutableRows = false;
boolean multiTenant = false;
Integer saltBucketNum = null;
String defaultFamilyName = null;
@@ -998,7 +1019,6 @@ public class MetaDataClient {
}
multiTenant = parent.isMultiTenant();
- isParentImmutableRows = parent.isImmutableRows();
parentTableName = parent.getTableName().getString();
// Pass through data table sequence number so we can check it hasn't changed
PreparedStatement incrementStatement = connection.prepareStatement(INCREMENT_SEQ_NUM);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c64cef6d/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index cdda1f1..b1f90a3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -553,7 +553,7 @@ public abstract class BaseTest {
@Override
public void run() {
try {
- utility.shutdownMiniCluster();
+ if (utility != null) utility.shutdownMiniCluster();
} catch (Exception e) {
logger.warn("Exception caught when shutting down mini cluster", e);
}