You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/12/30 19:51:57 UTC
phoenix git commit: PHOENIX-1511 Invalidate stats row even if not
running stats due to phoenix.stats.minUpdateFrequency
Repository: phoenix
Updated Branches:
refs/heads/master d6a4caf5c -> 9fc37c7cd
PHOENIX-1511 Invalidate stats row even if not running stats due to phoenix.stats.minUpdateFrequency
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9fc37c7c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9fc37c7c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9fc37c7c
Branch: refs/heads/master
Commit: 9fc37c7cd0ce33337c5fd0b133bf43bd7024d6d7
Parents: d6a4caf
Author: James Taylor <jt...@salesforce.com>
Authored: Tue Dec 30 10:51:48 2014 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Tue Dec 30 10:51:48 2014 -0800
----------------------------------------------------------------------
.../phoenix/end2end/StatsCollectorIT.java | 60 ++++++++++++++++----
.../apache/phoenix/schema/MetaDataClient.java | 33 ++++++-----
2 files changed, 66 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9fc37c7c/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index ca8ab81..8b3bdee 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -21,6 +21,7 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.apache.phoenix.util.TestUtil.getAllSplits;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -43,6 +44,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -228,28 +230,39 @@ public class StatsCollectorIT extends BaseOwnClusterHBaseManagedTimeIT {
return stmt;
}
- private void compactTable(Connection conn) throws IOException, InterruptedException, SQLException {
+ private void compactTable(Connection conn, String tableName) throws IOException, InterruptedException, SQLException {
ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
HBaseAdmin admin = services.getAdmin();
try {
- admin.flush(STATS_TEST_TABLE_NAME);
- admin.majorCompact(STATS_TEST_TABLE_NAME);
+ admin.flush(tableName);
+ admin.majorCompact(tableName);
Thread.sleep(10000); // FIXME: how do we know when compaction is done?
} finally {
admin.close();
}
- services.clearCache();
}
@Test
public void testCompactUpdatesStats() throws Exception {
+ testCompactUpdatesStats(null, STATS_TEST_TABLE_NAME + 1);
+ }
+
+ @Test
+ public void testCompactUpdatesStatsWithMinStatsUpdateFreq() throws Exception {
+ testCompactUpdatesStats(QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS, STATS_TEST_TABLE_NAME + 2);
+ }
+
+ private void testCompactUpdatesStats(Integer minStatsUpdateFreq, String tableName) throws Exception {
int nRows = 10;
Connection conn;
PreparedStatement stmt;
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ if (minStatsUpdateFreq != null) {
+ props.setProperty(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, minStatsUpdateFreq.toString());
+ }
conn = DriverManager.getConnection(getUrl(), props);
- conn.createStatement().execute("CREATE TABLE " + STATS_TEST_TABLE_NAME + "(k CHAR(1) PRIMARY KEY, v INTEGER) " + HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE);
- stmt = conn.prepareStatement("UPSERT INTO " + STATS_TEST_TABLE_NAME + " VALUES(?,?)");
+ conn.createStatement().execute("CREATE TABLE " + tableName + "(k CHAR(1) PRIMARY KEY, v INTEGER) " + HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE);
+ stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?)");
for (int i = 0; i < nRows; i++) {
stmt.setString(1, Character.toString((char) ('a' + i)));
stmt.setInt(2, i);
@@ -257,18 +270,41 @@ public class StatsCollectorIT extends BaseOwnClusterHBaseManagedTimeIT {
}
conn.commit();
- compactTable(conn);
- conn = DriverManager.getConnection(getUrl(), props);
- List<KeyRange>keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME);
+ compactTable(conn, tableName);
+ if (minStatsUpdateFreq == null) {
+ conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ }
+ // Confirm that when we have a non zero MIN_STATS_UPDATE_FREQ_MS_ATTRIB, after we run
+ // UPDATATE STATISTICS, the new statistics are faulted in as expected.
+ if (minStatsUpdateFreq != null) {
+ List<KeyRange>keyRanges = getAllSplits(conn, tableName);
+ assertNotEquals(nRows+1, keyRanges.size());
+ // If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
+ // and forcing the new stats to be pulled over.
+ int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName);
+ assertEquals(0, rowCount);
+ }
+ List<KeyRange>keyRanges = getAllSplits(conn, tableName);
assertEquals(nRows+1, keyRanges.size());
- int nDeletedRows = conn.createStatement().executeUpdate("DELETE FROM " + STATS_TEST_TABLE_NAME + " WHERE V < 5");
+ int nDeletedRows = conn.createStatement().executeUpdate("DELETE FROM " + tableName + " WHERE V < 5");
conn.commit();
assertEquals(5, nDeletedRows);
- compactTable(conn);
+ compactTable(conn, tableName);
+ if (minStatsUpdateFreq == null) {
+ conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ }
- keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME);
+ keyRanges = getAllSplits(conn, tableName);
+ if (minStatsUpdateFreq != null) {
+ assertEquals(nRows+1, keyRanges.size());
+ // If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
+ // and force us to pull over the new stats
+ int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName);
+ assertEquals(0, rowCount);
+ keyRanges = getAllSplits(conn, tableName);
+ }
assertEquals(nRows/2+1, keyRanges.size());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9fc37c7c/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 5b1315f..623fc0e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -658,25 +658,28 @@ public class MetaDataClient {
if (rs.next()) {
msSinceLastUpdate = rs.getLong(1) - rs.getLong(2);
}
- if (msSinceLastUpdate < msMinBetweenUpdates) {
- return 0;
+ long rowCount = 0;
+ if (msSinceLastUpdate >= msMinBetweenUpdates) {
+ /*
+ * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through,
+ * since it may not represent a "real" table in the case of the view indexes of a base table.
+ */
+ PostDDLCompiler compiler = new PostDDLCompiler(connection);
+ TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false);
+ MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
+ Scan scan = plan.getContext().getScan();
+ scan.setCacheBlocks(false);
+ scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
+ MutationState mutationState = plan.execute();
+ rowCount = mutationState.getUpdateCount();
}
/*
- * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through,
- * since it may not represent a "real" table in the case of the view indexes of a base table.
+ * Update the stats table so that client will pull the new one with the updated stats.
+ * Even if we don't run the command due to the last update time, invalidate the cache.
+ * This supports scenarios in which a major compaction was manually initiated and the
+ * client wants the modified stats to be reflected immediately.
*/
- PostDDLCompiler compiler = new PostDDLCompiler(connection);
- TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false);
- MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
- Scan scan = plan.getContext().getScan();
- scan.setCacheBlocks(false);
- scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
- MutationState mutationState = plan.execute();
- long rowCount = mutationState.getUpdateCount();
-
- // We need to update the stats table so that client will pull the new one with
- // the updated stats.
connection.getQueryServices().clearTableFromCache(tenantIdBytes,
Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);