You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2016/10/10 20:11:53 UTC
[3/3] phoenix git commit: PHOENIX-3361 Collect stats correct for
local indexes
PHOENIX-3361 Collect stats correct for local indexes
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a5b710d1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a5b710d1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a5b710d1
Branch: refs/heads/4.x-HBase-1.1
Commit: a5b710d1fe8c8f858f8659981a0b00ba9feddef2
Parents: 812be0b
Author: James Taylor <ja...@apache.org>
Authored: Thu Oct 6 22:27:10 2016 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Mon Oct 10 13:14:35 2016 -0700
----------------------------------------------------------------------
.../apache/phoenix/end2end/GroupByCaseIT.java | 7 +-
.../apache/phoenix/end2end/IndexExtendedIT.java | 7 +-
.../phoenix/end2end/RoundFloorCeilFuncIT.java | 1 -
.../phoenix/end2end/StatsCollectorIT.java | 53 +++--
.../end2end/index/MutableIndexFailureIT.java | 2 -
.../phoenix/end2end/index/MutableIndexIT.java | 1 -
.../phoenix/end2end/index/ViewIndexIT.java | 85 +++----
.../UngroupedAggregateRegionObserver.java | 8 +-
.../phoenix/iterate/BaseResultIterators.java | 84 +++----
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 6 +-
.../phoenix/query/ConnectionQueryServices.java | 20 +-
.../query/ConnectionQueryServicesImpl.java | 55 +++--
.../query/ConnectionlessQueryServicesImpl.java | 28 +--
.../query/DelegateConnectionQueryServices.java | 12 +-
.../apache/phoenix/query/GuidePostsCache.java | 231 +++++++++++++++++++
.../apache/phoenix/query/QueryConstants.java | 2 +-
.../apache/phoenix/query/TableStatsCache.java | 192 ---------------
.../apache/phoenix/schema/MetaDataClient.java | 49 ++--
.../stats/DefaultStatisticsCollector.java | 44 ++--
.../phoenix/schema/stats/GuidePostsInfo.java | 77 ++++---
.../schema/stats/GuidePostsInfoBuilder.java | 4 +
.../phoenix/schema/stats/GuidePostsKey.java | 84 +++++++
.../schema/stats/NoOpStatisticsCollector.java | 3 +-
.../phoenix/schema/stats/PTableStats.java | 57 -----
.../phoenix/schema/stats/PTableStatsImpl.java | 115 ---------
.../schema/stats/StatisticsCollector.java | 5 +-
.../phoenix/schema/stats/StatisticsUtil.java | 96 ++++----
.../phoenix/schema/stats/StatisticsWriter.java | 6 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 4 +
.../phoenix/filter/SkipScanBigFilterTest.java | 25 +-
.../PhoenixStatsCacheRemovalListenerTest.java | 2 +-
.../java/org/apache/phoenix/util/TestUtil.java | 28 ++-
32 files changed, 697 insertions(+), 696 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index 7accf37..b842e36 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -33,6 +33,7 @@ import java.sql.Statement;
import java.util.List;
import java.util.Properties;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.AmbiguousColumnException;
import org.apache.phoenix.schema.types.PChar;
@@ -409,11 +410,11 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
String tableName = generateUniqueName();
- initAvgGroupTable(conn, tableName, " GUIDE_POST_WIDTH=20 ");
+ initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 ");
testAvgGroupByOrderPreserving(conn, tableName, 13);
- conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=" + 100);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=100");
testAvgGroupByOrderPreserving(conn, tableName, 6);
- conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=null");
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=null");
testAvgGroupByOrderPreserving(conn, tableName, 4);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
index cfbc9eb..93d7cec 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexExtendedIT.java
@@ -92,10 +92,11 @@ public class IndexExtendedIT extends BaseOwnClusterIT {
@BeforeClass
public static void doSetup() throws Exception {
- Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(1);
+ Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
- Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(1);
+ Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
+ clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet()
.iterator()));
}
@@ -442,7 +443,7 @@ public class IndexExtendedIT extends BaseOwnClusterIT {
rs = conn1.createStatement().executeQuery(query);
Thread.sleep(1000);
for (int j = 0; j < 26; j++) {
- assertTrue(rs.next());
+ assertTrue("No row found at " + j, rs.next());
tIdColumnValues[j] = rs.getString("t_id");
k1ColumnValue[j] = rs.getInt("k1");
v1ColumnValues[j] = rs.getString("V1");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
index c247bc9..9961199 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RoundFloorCeilFuncIT.java
@@ -175,7 +175,6 @@ public class RoundFloorCeilFuncIT extends ParallelStatsDisabledIT {
expectedDate = DateUtil.parseDate("2012-01-01 14:25:29");
assertEquals(expectedDate, rs.getDate(4));
expectedDate = DateUtil.parseDate("2012-01-02 00:00:00");
- System.out.println(String.format(" the expected time is [%s] and the actual time is [%s]",expectedDate.getTime(),rs.getDate(5).getTime()));
assertEquals(expectedDate, rs.getDate(5));
expectedDate = DateUtil.parseDate("2012-02-01 00:00:00");
assertEquals(expectedDate, rs.getDate(6));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
index 2445948..6193cad 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -40,13 +40,15 @@ import java.util.Random;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
@@ -69,10 +71,14 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
}
private static Connection getConnection() throws SQLException {
+ return getConnection(Integer.MAX_VALUE);
+ }
+
+ private static Connection getConnection(Integer statsUpdateFreq) throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
props.setProperty(QueryServices.EXPLAIN_CHUNK_COUNT_ATTRIB, Boolean.TRUE.toString());
props.setProperty(QueryServices.EXPLAIN_ROW_COUNT_ATTRIB, Boolean.TRUE.toString());
- props.setProperty(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
+ props.setProperty(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Integer.toString(statsUpdateFreq));
return DriverManager.getConnection(getUrl(), props);
}
@@ -334,7 +340,7 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
@Test
public void testCompactUpdatesStats() throws Exception {
- testCompactUpdatesStats(null, fullTableName);
+ testCompactUpdatesStats(0, fullTableName);
}
@Test
@@ -342,9 +348,17 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
testCompactUpdatesStats(QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS, fullTableName);
}
- private void testCompactUpdatesStats(Integer minStatsUpdateFreq, String tableName) throws Exception {
+ private static void invalidateStats(Connection conn, String tableName) throws SQLException {
+ PTable ptable = conn.unwrap(PhoenixConnection.class)
+ .getMetaDataCache().getTableRef(new PTableKey(null, tableName))
+ .getTable();
+ byte[] name = ptable.getPhysicalName().getBytes();
+ conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(new GuidePostsKey(name, SchemaUtil.getEmptyColumnFamily(ptable)));
+ }
+
+ private void testCompactUpdatesStats(Integer statsUpdateFreq, String tableName) throws Exception {
int nRows = 10;
- Connection conn = getConnection();
+ Connection conn = getConnection(statsUpdateFreq);
PreparedStatement stmt;
conn.createStatement().execute("CREATE TABLE " + tableName + "(k CHAR(1) PRIMARY KEY, v INTEGER, w INTEGER) "
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE);
@@ -358,9 +372,8 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
conn.commit();
compactTable(conn, tableName);
- if (minStatsUpdateFreq == null) {
- ImmutableBytesPtr ptr = new ImmutableBytesPtr(Bytes.toBytes(tableName));
- conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(ptr);
+ if (statsUpdateFreq == null) {
+ invalidateStats(conn, tableName);
} else {
// Confirm that when we have a non zero MIN_STATS_UPDATE_FREQ_MS_ATTRIB, after we run
// UPDATATE STATISTICS, the new statistics are faulted in as expected.
@@ -379,13 +392,12 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
assertEquals(5, nDeletedRows);
compactTable(conn, tableName);
- if (minStatsUpdateFreq == null) {
- ImmutableBytesPtr ptr = new ImmutableBytesPtr(Bytes.toBytes(tableName));
- conn.unwrap(PhoenixConnection.class).getQueryServices().invalidateStats(ptr);
+ if (statsUpdateFreq == null) {
+ invalidateStats(conn, tableName);
}
keyRanges = getAllSplits(conn, tableName);
- if (minStatsUpdateFreq != null) {
+ if (statsUpdateFreq != null) {
assertEquals(nRows+1, keyRanges.size());
// If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache
// and force us to pull over the new stats
@@ -403,7 +415,7 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
@Test
public void testWithMultiCF() throws Exception {
int nRows = 20;
- Connection conn = getConnection();
+ Connection conn = getConnection(0);
PreparedStatement stmt;
conn.createStatement().execute(
"CREATE TABLE " + fullTableName
@@ -478,6 +490,20 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
assertEquals(6, rs.getInt(4));
assertFalse(rs.next());
+
+ // Disable stats
+ conn.createStatement().execute("ALTER TABLE " + fullTableName +
+ " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=0");
+ TestUtil.analyzeTable(conn, fullTableName);
+ // Assert that there are no more guideposts
+ rs = conn.createStatement().executeQuery("SELECT count(1) FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME +
+ " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + "='" + fullTableName + "' AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS NOT NULL");
+ assertTrue(rs.next());
+ assertEquals(0, rs.getLong(1));
+ assertFalse(rs.next());
+ rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName);
+ assertEquals("CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN OVER " + fullTableName,
+ QueryUtil.getExplainPlan(rs));
}
@Test
@@ -506,7 +532,6 @@ public class StatsCollectorIT extends ParallelStatsEnabledIT {
int endIndex = r.nextInt(strings.length - startIndex) + startIndex;
long rows = endIndex - startIndex;
long c2Bytes = rows * 35;
- System.out.println(rows + ":" + startIndex + ":" + endIndex);
rs = conn.createStatement().executeQuery(
"SELECT COLUMN_FAMILY,SUM(GUIDE_POSTS_ROW_COUNT),SUM(GUIDE_POSTS_WIDTH) from SYSTEM.STATS where PHYSICAL_NAME = '"
+ fullTableName + "' AND GUIDE_POST_KEY>= cast('" + strings[startIndex]
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index d740013..d6c1e9c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -377,9 +377,7 @@ public class MutableIndexFailureIT extends BaseOwnClusterIT {
conn.commit();
fail();
} catch (SQLException e) {
- System.out.println();
} catch (Exception e) {
- System.out.println();
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index 7727ba3..6531d95 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -703,7 +703,6 @@ public class MutableIndexIT extends ParallelStatsDisabledIT {
while (true) {
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
- System.out.println("Number of rows returned:" + rs.getInt(1));
assertEquals(4, rs.getInt(1)); //TODO this returns 5 sometimes instead of 4, duplicate results?
try {
List<HRegionInfo> indexRegions = admin.getTableRegions(indexTable);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
index b714a11..9e63093 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
@@ -31,11 +31,14 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.List;
import java.util.Properties;
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PNameFactory;
import org.apache.phoenix.schema.PTableType;
@@ -145,19 +148,19 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
stmt.setString(4, "x1");
stmt.setInt(5, 100);
stmt.execute();
- stmt.setString(1, "20");
+ stmt.setString(1, "10");
stmt.setString(2, "b");
stmt.setInt(3, 2);
stmt.setString(4, "x2");
stmt.setInt(5, 200);
stmt.execute();
- stmt.setString(1, "30");
+ stmt.setString(1, "10");
stmt.setString(2, "c");
stmt.setInt(3, 3);
stmt.setString(4, "x3");
stmt.setInt(5, 300);
stmt.execute();
- stmt.setString(1, "40");
+ stmt.setString(1, "20");
stmt.setString(2, "d");
stmt.setInt(3, 4);
stmt.setString(4, "x4");
@@ -185,24 +188,24 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
assertTrue(rs.next());
assertFalse(rs.next());
-// TestUtil.analyzeTable(conn, fullTableName);
-// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName);
-// assertEquals(1, guideposts.size());
-// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
-//
-// conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET GUIDE_POST_WIDTH=20");
-//
-// TestUtil.analyzeTable(conn, fullTableName);
-// guideposts = TestUtil.getAllSplits(conn, fullTableName);
-// assertEquals(5, guideposts.size());
-//
-// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
-// // was used
-// sql = "SELECT * FROM " + viewName + " WHERE v2 > 100";
-// stmt = conn1.prepareStatement(sql);
-// stmt.executeQuery();
-// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
-// assertEquals(5, plan.getSplits().size());
+ TestUtil.analyzeTable(conn, fullTableName);
+ List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName);
+ assertEquals(1, guideposts.size());
+ assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+
+ conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20");
+
+ TestUtil.analyzeTable(conn, fullTableName);
+ guideposts = TestUtil.getAllSplits(conn, fullTableName);
+ assertEquals(5, guideposts.size());
+
+ // Confirm that when view index used, the GUIDE_POSTS_WIDTH from the data physical table
+ // was used
+ sql = "SELECT * FROM " + viewName + " WHERE v2 >= 100";
+ stmt = conn1.prepareStatement(sql);
+ stmt.executeQuery();
+ QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+ assertEquals(4, plan.getSplits().size());
}
@@ -259,26 +262,26 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
assertEquals("KV1", rs.getString(1));
assertFalse(rs.next());
-// TestUtil.analyzeTable(conn, baseTable);
-// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable);
-// assertEquals(1, guideposts.size());
-// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
-//
-// conn.createStatement().execute("ALTER TABLE " + baseTable + " SET GUIDE_POST_WIDTH=20");
-//
-// TestUtil.analyzeTable(conn, baseTable);
-// guideposts = TestUtil.getAllSplits(conn, baseTable);
-// assertEquals(6, guideposts.size());
-//
-// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
-// // was used
-// stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?");
-// stmt.setInt(1, 1);
-// stmt.setString(2, "KV3");
-// rs = stmt.executeQuery();
-// plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
-// assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx));
-// assertEquals(6, plan.getSplits().size());
+ TestUtil.analyzeTable(conn, baseTable);
+ List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable);
+ assertEquals(1, guideposts.size());
+ assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+
+ conn.createStatement().execute("ALTER TABLE " + baseTable + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20");
+
+ TestUtil.analyzeTable(conn, baseTable);
+ guideposts = TestUtil.getAllSplits(conn, baseTable);
+ assertEquals(6, guideposts.size());
+
+ // Confirm that when view index used, the GUIDE_POSTS_WIDTH from the data physical table
+ // was used
+ stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?");
+ stmt.setInt(1, 1);
+ stmt.setString(2, "KV3");
+ rs = stmt.executeQuery();
+ plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+ assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx));
+ assertEquals(6, plan.getSplits().size());
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 2a4bfca..d8d313d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -718,7 +718,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
private RegionScanner collectStats(final RegionScanner innerScanner, StatisticsCollector stats,
final Region region, final Scan scan, Configuration config) throws IOException {
StatsCollectionCallable callable =
- new StatsCollectionCallable(stats, region, innerScanner, config);
+ new StatsCollectionCallable(stats, region, innerScanner, config, scan);
byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB);
boolean async = false;
if (asyncBytes != null) {
@@ -785,13 +785,15 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
private final Region region;
private final RegionScanner innerScanner;
private final Configuration config;
+ private final Scan scan;
StatsCollectionCallable(StatisticsCollector s, Region r, RegionScanner rs,
- Configuration config) {
+ Configuration config, Scan scan) {
this.stats = s;
this.region = r;
this.innerScanner = rs;
this.config = config;
+ this.scan = scan;
}
@Override
@@ -832,7 +834,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
} finally {
try {
if (noErrors && !compactionRunning) {
- stats.updateStatistic(region);
+ stats.updateStatistic(region, scan);
logger.info("UPDATE STATISTICS finished successfully for scanner: "
+ innerScanner + ". Number of rows scanned: " + rowCount
+ ". Time: " + (System.currentTimeMillis() - startTime));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 2685b93..d4c8bef 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -86,7 +86,7 @@ import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.stats.GuidePostsInfo;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.schema.stats.StatisticsUtil;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.Closeables;
@@ -120,7 +120,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
private final List<List<Scan>> scans;
private final List<KeyRange> splits;
- private final PTableStats tableStats;
private final byte[] physicalTableName;
protected final QueryPlan plan;
protected final String scanId;
@@ -363,9 +362,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
if (null == currentSCN) {
currentSCN = HConstants.LATEST_TIMESTAMP;
}
- tableStats = useStats() && StatisticsUtil.isStatsEnabled(TableName.valueOf(physicalTableName))
- ? context.getConnection().getQueryServices().getTableStats(physicalTableName, currentSCN)
- : PTableStats.EMPTY_STATS;
// Used to tie all the scans together during logging
scanId = new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong()).toString();
@@ -425,55 +421,40 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return guideIndex;
}
- private GuidePostsInfo getGuidePosts(Set<byte[]> whereConditions) {
- if (!useStats()) { return GuidePostsInfo.NO_GUIDEPOST; }
+ private GuidePostsInfo getGuidePosts() throws SQLException {
+ if (!useStats() || !StatisticsUtil.isStatsEnabled(TableName.valueOf(physicalTableName))) {
+ return GuidePostsInfo.NO_GUIDEPOST;
+ }
- GuidePostsInfo gps = null;
+ TreeSet<byte[]> whereConditions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+ for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
+ byte[] cf = where.getFirst();
+ if (cf != null) {
+ whereConditions.add(cf);
+ }
+ }
PTable table = getTable();
- Map<byte[], GuidePostsInfo> guidePostMap = tableStats.getGuidePosts();
byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable());
- if (table.getColumnFamilies().isEmpty()) {
- // For sure we can get the defaultCF from the table
- gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
- } else {
- if (whereConditions.isEmpty() || whereConditions.contains(defaultCF)) {
- gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
- } else {
- byte[] familyInWhere = whereConditions.iterator().next();
- GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere);
- if (guidePostsInfo != null) {
- gps = guidePostsInfo;
- } else {
- // As there are no guideposts collected for the where family we go with the default CF
- gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF);
+ byte[] cf = null;
+ if ( !table.getColumnFamilies().isEmpty() && !whereConditions.isEmpty() ) {
+ for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
+ byte[] whereCF = where.getFirst();
+ if (Bytes.compareTo(defaultCF, whereCF) == 0) {
+ cf = defaultCF;
+ break;
}
}
+ if (cf == null) {
+ cf = context.getWhereConditionColumns().get(0).getFirst();
+ }
}
- if (gps == null) { return GuidePostsInfo.NO_GUIDEPOST; }
- return gps;
- }
-
- private GuidePostsInfo getDefaultFamilyGuidePosts(Map<byte[], GuidePostsInfo> guidePostMap, byte[] defaultCF) {
- if (guidePostMap.get(defaultCF) != null) {
- return guidePostMap.get(defaultCF);
+ if (cf == null) {
+ cf = defaultCF;
}
- return null;
+ GuidePostsKey key = new GuidePostsKey(physicalTableName, cf);
+ return context.getConnection().getQueryServices().getTableStats(key);
}
- private static String toString(List<byte[]> gps) {
- StringBuilder buf = new StringBuilder(gps.size() * 100);
- buf.append("[");
- for (int i = 0; i < gps.size(); i++) {
- buf.append(Bytes.toStringBinary(gps.get(i)));
- buf.append(",");
- if (i > 0 && i < gps.size()-1 && (i % 10) == 0) {
- buf.append("\n");
- }
- }
- buf.setCharAt(buf.length()-1, ']');
- return buf.toString();
- }
-
private List<Scan> addNewScan(List<List<Scan>> parallelScans, List<Scan> scans, Scan scan, byte[] startKey, boolean crossedRegionBoundary, HRegionLocation regionLocation) {
boolean startNewScan = scanGrouper.shouldStartNewScan(plan, scans, startKey, crossedRegionBoundary);
if (scan != null) {
@@ -565,14 +546,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
PTable table = getTable();
boolean isSalted = table.getBucketNum() != null;
boolean isLocalIndex = table.getIndexType() == IndexType.LOCAL;
- TreeSet<byte[]> whereConditions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for(Pair<byte[], byte[]> where : context.getWhereConditionColumns()) {
- byte[] cf = where.getFirst();
- if (cf != null) {
- whereConditions.add(cf);
- }
- }
- GuidePostsInfo gps = getGuidePosts(whereConditions);
+ GuidePostsInfo gps = getGuidePosts();
hasGuidePosts = gps != GuidePostsInfo.NO_GUIDEPOST;
boolean traverseAllRegions = isSalted || isLocalIndex;
if (!traverseAllRegions) {
@@ -649,8 +623,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
if (newScan != null) {
ScanUtil.setLocalIndexAttributes(newScan, keyOffset, regionInfo.getStartKey(),
regionInfo.getEndKey(), newScan.getStartRow(), newScan.getStopRow());
- estimatedRows += gps.getRowCounts().get(guideIndex);
- estimatedSize += gps.getByteCounts().get(guideIndex);
+ estimatedRows += gps.getRowCounts()[guideIndex];
+ estimatedSize += gps.getByteCounts()[guideIndex];
}
scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation);
currentKeyBytes = currentGuidePostBytes;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 47dfd4e..35b754f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -1087,7 +1087,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
SQLViewTypeFunction.NAME + "(" + VIEW_TYPE + ") AS " + VIEW_TYPE + "," +
SQLIndexTypeFunction.NAME + "(" + INDEX_TYPE + ") AS " + INDEX_TYPE + "," +
TRANSACTIONAL + "," +
- IS_NAMESPACE_MAPPED +
+ IS_NAMESPACE_MAPPED + "," +
+ GUIDE_POSTS_WIDTH +
" from " + SYSTEM_CATALOG + " " + SYSTEM_CATALOG_ALIAS +
" where " + COLUMN_NAME + " is null" +
" and " + COLUMN_FAMILY + " is null" +
@@ -1126,7 +1127,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
"'' " + VIEW_TYPE + "," +
"'' " + INDEX_TYPE + "," +
"CAST(null AS BOOLEAN) " + TRANSACTIONAL + "," +
- "CAST(null AS BOOLEAN) " + IS_NAMESPACE_MAPPED + "\n");
+ "CAST(null AS BOOLEAN) " + IS_NAMESPACE_MAPPED + "," +
+ "CAST(null AS BIGINT) " + GUIDE_POSTS_WIDTH + "\n");
buf.append(
" from " + SYSTEM_SEQUENCE + "\n");
StringBuilder whereClause = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index bc2c93b..0478e07 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.compile.MutationPlan;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.execute.MutationState;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.PFunction;
@@ -43,7 +42,8 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.tephra.TransactionSystemClient;
@@ -118,7 +118,14 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public String getUserName();
public void clearTableFromCache(final byte[] tenantId, final byte[] schemaName, final byte[] tableName, long clientTS) throws SQLException;
- public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) throws SQLException;
+ public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException;
+ /**
+ * Removes cache {@link GuidePostsInfo} for the table with the given name. If no cached guideposts are present, this does nothing.
+ *
+ * @param tableName The table to remove stats for
+ */
+ void invalidateStats(GuidePostsKey key);
+
public long clearCache() throws SQLException;
public int getSequenceSaltBuckets();
@@ -133,13 +140,6 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public MetaDataMutationResult dropSchema(List<Mutation> schemaMetaData, String schemaName) throws SQLException;
- /**
- * Removes cache {@link PTableStats} for the table with the given name. If no cached stats are present, this does nothing.
- *
- * @param tableName The table to remove stats for
- */
- void invalidateStats(ImmutableBytesPtr tableName);
-
boolean isUpgradeRequired();
void upgradeSystemTables(String url, Properties props) throws SQLException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 51526cd..bad3178 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -140,7 +140,6 @@ import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy;
import org.apache.phoenix.hbase.index.Indexer;
import org.apache.phoenix.hbase.index.covered.NonTxIndexBuilder;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.hbase.index.util.VersionUtil;
import org.apache.phoenix.index.PhoenixIndexBuilder;
@@ -179,7 +178,8 @@ import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.TableProperty;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PInteger;
@@ -235,7 +235,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private final ReadOnlyProps props;
private final String userName;
private final ConcurrentHashMap<ImmutableBytesWritable,ConnectionQueryServices> childServices;
- private final TableStatsCache tableStatsCache;
+ private final GuidePostsCache tableStatsCache;
// Cache the latest meta data here for future connections
// writes guarded by "latestMetaDataLock"
@@ -348,7 +348,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
connectionQueues = ImmutableList.copyOf(list);
// A little bit of a smell to leak `this` here, but should not be a problem
- this.tableStatsCache = new TableStatsCache(this, config);
+ this.tableStatsCache = new GuidePostsCache(this, config);
this.isAutoUpgradeEnabled = config.getBoolean(AUTO_UPGRADE_ENABLED, QueryServicesOptions.DEFAULT_AUTO_UPGRADE_ENABLED);
}
@@ -1317,13 +1317,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private boolean ensureViewIndexTableDropped(byte[] physicalTableName, long timestamp) throws SQLException {
byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName);
- HTableDescriptor desc = null;
boolean wasDeleted = false;
try (HBaseAdmin admin = getAdmin()) {
try {
- desc = admin.getTableDescriptor(physicalIndexName);
+ HTableDescriptor desc = admin.getTableDescriptor(physicalIndexName);
if (Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES)))) {
- this.tableStatsCache.invalidate(new ImmutableBytesPtr(physicalIndexName));
final ReadOnlyProps props = this.getProps();
final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
if (dropMetadata) {
@@ -1331,6 +1329,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
admin.deleteTable(physicalIndexName);
clearTableRegionCache(physicalIndexName);
wasDeleted = true;
+ } else {
+ this.tableStatsCache.invalidateAll(desc);
}
}
} catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
@@ -1348,7 +1348,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try (HBaseAdmin admin = getAdmin()) {
try {
desc = admin.getTableDescriptor(physicalTableName);
- this.tableStatsCache.invalidate(new ImmutableBytesPtr(physicalTableName));
+ for (byte[] fam : desc.getFamiliesKeys()) {
+ this.tableStatsCache.invalidate(new GuidePostsKey(physicalTableName, fam));
+ }
final ReadOnlyProps props = this.getProps();
final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
if (dropMetadata) {
@@ -1530,14 +1532,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (dropMetadata) {
flushParentPhysicalTable(table);
dropTables(result.getTableNamesToDelete());
+ } else {
+ invalidateTableStats(result.getTableNamesToDelete());
}
- invalidateTables(result.getTableNamesToDelete());
if (tableType == PTableType.TABLE) {
long timestamp = MetaDataUtil.getClientTimeStamp(tableMetaData);
byte[] physicalName = table.getPhysicalName().getBytes();
ensureViewIndexTableDropped(physicalName, timestamp);
ensureLocalIndexTableDropped(physicalName, timestamp);
- tableStatsCache.invalidate(new ImmutableBytesPtr(physicalName));
+ tableStatsCache.invalidateAll(table);
}
break;
default:
@@ -1598,24 +1601,30 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return result;
}
- private void invalidateTables(final List<byte[]> tableNamesToDelete) {
+ private void invalidateTableStats(final List<byte[]> tableNamesToDelete) {
if (tableNamesToDelete != null) {
for (byte[] tableName : tableNamesToDelete) {
- tableStatsCache.invalidate(new ImmutableBytesPtr(Bytes.toString(tableName)
- .replace(QueryConstants.NAMESPACE_SEPARATOR, QueryConstants.NAME_SEPARATOR).getBytes()));
+ tableStatsCache.invalidateAll(tableName);
}
}
}
+ private void dropTable(byte[] tableNameToDelete) throws SQLException {
+ dropTables(Collections.<byte[]>singletonList(tableNameToDelete));
+ }
+
private void dropTables(final List<byte[]> tableNamesToDelete) throws SQLException {
SQLException sqlE = null;
try (HBaseAdmin admin = getAdmin()) {
if (tableNamesToDelete != null){
for ( byte[] tableName : tableNamesToDelete ) {
- if ( admin.tableExists(tableName) ) {
+ try {
+ HTableDescriptor htableDesc = this.getTableDescriptor(tableName);
admin.disableTable(tableName);
admin.deleteTable(tableName);
+ tableStatsCache.invalidateAll(htableDesc);
clearTableRegionCache(tableName);
+ } catch (TableNotFoundException ignore) {
}
}
}
@@ -2248,8 +2257,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
if (dropMetadata) {
dropTables(result.getTableNamesToDelete());
+ } else {
+ invalidateTableStats(result.getTableNamesToDelete());
}
- invalidateTables(result.getTableNamesToDelete());
break;
default:
break;
@@ -3478,7 +3488,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
sqlE = new SQLException(e);
} finally {
try {
- if (tenantId.length == 0) tableStatsCache.invalidate(new ImmutableBytesPtr(SchemaUtil.getTableNameAsBytes(schemaName, tableName)));
htable.close();
} catch (IOException e) {
if (sqlE == null) {
@@ -3680,9 +3689,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
@Override
- public PTableStats getTableStats(final byte[] physicalName, final long clientTimeStamp) throws SQLException {
+ public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException {
try {
- return tableStatsCache.get(new ImmutableBytesPtr(physicalName));
+ return tableStatsCache.get(key);
} catch (ExecutionException e) {
throw ServerUtil.parseServerException(e);
}
@@ -4042,19 +4051,19 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
/**
- * Manually adds {@link PTableStats} for a table to the client-side cache. Not a
+ * Manually adds {@link GuidePostsInfo} for a table to the client-side cache. Not a
* {@link ConnectionQueryServices} method. Exposed for testing purposes.
*
* @param tableName Table name
* @param stats Stats instance
*/
- public void addTableStats(ImmutableBytesPtr tableName, PTableStats stats) {
- this.tableStatsCache.put(Objects.requireNonNull(tableName), stats);
+ public void addTableStats(GuidePostsKey key, GuidePostsInfo info) {
+ this.tableStatsCache.put(Objects.requireNonNull(key), Objects.requireNonNull(info));
}
@Override
- public void invalidateStats(ImmutableBytesPtr tableName) {
- this.tableStatsCache.invalidate(Objects.requireNonNull(tableName));
+ public void invalidateStats(GuidePostsKey key) {
+ this.tableStatsCache.invalidate(Objects.requireNonNull(key));
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 8c9f3ff..1b1e429 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -51,7 +51,6 @@ import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
@@ -79,7 +78,8 @@ import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SequenceNotFoundException;
import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.util.JDBCUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -113,7 +113,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
private volatile boolean initialized;
private volatile SQLException initializationException;
private final Map<String, List<HRegionLocation>> tableSplits = Maps.newHashMap();
- private final TableStatsCache tableStatsCache;
+ private final GuidePostsCache guidePostsCache;
public ConnectionlessQueryServicesImpl(QueryServices services, ConnectionInfo connInfo, Properties info) {
super(services);
@@ -140,7 +140,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(config);
TransactionManager txnManager = new TransactionManager(config);
this.txSystemClient = new InMemoryTxSystemClient(txnManager);
- this.tableStatsCache = new TableStatsCache(this, config);
+ this.guidePostsCache = new GuidePostsCache(this, config);
}
private PMetaData newEmptyMetaData() {
@@ -520,12 +520,12 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) {
- PTableStats stats = tableStatsCache.getCache().getIfPresent(physicalName);
- if (null == stats) {
- return PTableStats.EMPTY_STATS;
+ public GuidePostsInfo getTableStats(GuidePostsKey key) {
+ GuidePostsInfo info = guidePostsCache.getCache().getIfPresent(key);
+ if (null == info) {
+ return GuidePostsInfo.NO_GUIDEPOST;
}
- return stats;
+ return info;
}
@Override
@@ -640,19 +640,19 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
/**
- * Manually adds {@link PTableStats} for a table to the client-side cache. Not a
+ * Manually adds {@link GuidePostsInfo} for a table to the client-side cache. Not a
* {@link ConnectionQueryServices} method. Exposed for testing purposes.
*
* @param tableName Table name
* @param stats Stats instance
*/
- public void addTableStats(ImmutableBytesPtr tableName, PTableStats stats) {
- this.tableStatsCache.put(Objects.requireNonNull(tableName), stats);
+ public void addTableStats(GuidePostsKey key, GuidePostsInfo info) {
+ this.guidePostsCache.put(Objects.requireNonNull(key), info);
}
@Override
- public void invalidateStats(ImmutableBytesPtr tableName) {
- this.tableStatsCache.invalidate(Objects.requireNonNull(tableName));
+ public void invalidateStats(GuidePostsKey key) {
+ this.guidePostsCache.invalidate(Objects.requireNonNull(key));
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index b00e92b..7466e9c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.compile.MutationPlan;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.execute.MutationState;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.PFunction;
@@ -45,7 +44,8 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceKey;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.tephra.TransactionSystemClient;
@@ -250,8 +250,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public PTableStats getTableStats(byte[] physicalName, long clientTimeStamp) throws SQLException {
- return getDelegate().getTableStats(physicalName, clientTimeStamp);
+ public GuidePostsInfo getTableStats(GuidePostsKey key) throws SQLException {
+ return getDelegate().getTableStats(key);
}
@@ -342,8 +342,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public void invalidateStats(ImmutableBytesPtr tableName) {
- getDelegate().invalidateStats(tableName);
+ public void invalidateStats(GuidePostsKey key) {
+ getDelegate().invalidateStats(key);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java b/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java
new file mode 100644
index 0000000..d27be1b
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/GuidePostsCache.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalCause;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.cache.Weigher;
+
+/**
+ * "Client-side" cache for storing {@link GuidePostsInfo} for a column family. Intended to decouple
+ * Phoenix from a specific version of Guava's cache.
+ */
+public class GuidePostsCache {
+ private static final Logger logger = LoggerFactory.getLogger(GuidePostsCache.class);
+
+ private final ConnectionQueryServices queryServices;
+ private final LoadingCache<GuidePostsKey, GuidePostsInfo> cache;
+
+ public GuidePostsCache(ConnectionQueryServices queryServices, Configuration config) {
+ this.queryServices = Objects.requireNonNull(queryServices);
+ // Number of millis to expire cache values after write
+ final long statsUpdateFrequency = config.getLong(
+ QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS);
+ // Maximum number of entries (tables) to store in the cache at one time
+ final long maxTableStatsCacheSize = config.getLong(
+ QueryServices.STATS_MAX_CACHE_SIZE,
+ QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE);
+ cache = CacheBuilder.newBuilder()
+ // Expire entries a given amount of time after they were written
+ .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS)
+ // Maximum total weight (size in bytes) of stats entries
+ .maximumWeight(maxTableStatsCacheSize)
+ // Defer actual size to the PTableStats.getEstimatedSize()
+ .weigher(new Weigher<GuidePostsKey, GuidePostsInfo>() {
+ @Override public int weigh(GuidePostsKey key, GuidePostsInfo info) {
+ return info.getEstimatedSize();
+ }
+ })
+ // Log removals at TRACE for debugging
+ .removalListener(new PhoenixStatsCacheRemovalListener())
+ // Automatically load the cache when entries are missing
+ .build(new StatsLoader());
+ }
+
+ /**
+ * {@link CacheLoader} implementation for the Phoenix Table Stats cache.
+ */
+ protected class StatsLoader extends CacheLoader<GuidePostsKey, GuidePostsInfo> {
+ @Override
+ public GuidePostsInfo load(GuidePostsKey statsKey) throws Exception {
+ @SuppressWarnings("deprecation")
+ HTableInterface statsHTable = queryServices.getTable(SchemaUtil.getPhysicalName(
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,
+ queryServices.getProps()).getName());
+ try {
+ GuidePostsInfo guidePostsInfo = StatisticsUtil.readStatistics(statsHTable, statsKey,
+ HConstants.LATEST_TIMESTAMP);
+ traceStatsUpdate(statsKey, guidePostsInfo);
+ return guidePostsInfo;
+ } catch (TableNotFoundException e) {
+ // On a fresh install, stats might not yet be created, don't warn about this.
+ logger.debug("Unable to locate Phoenix stats table", e);
+ return GuidePostsInfo.NO_GUIDEPOST;
+ } catch (IOException e) {
+ logger.warn("Unable to read from stats table", e);
+ // Just cache empty stats. We'll try again after some time anyway.
+ return GuidePostsInfo.NO_GUIDEPOST;
+ } finally {
+ try {
+ statsHTable.close();
+ } catch (IOException e) {
+ // Log, but continue. We have our stats anyway now.
+ logger.warn("Unable to close stats table", e);
+ }
+ }
+ }
+
+ /**
+ * Logs a trace message for newly inserted entries to the stats cache.
+ */
+ void traceStatsUpdate(GuidePostsKey key, GuidePostsInfo info) {
+ if (logger.isTraceEnabled()) {
+ logger.trace("Updating local TableStats cache (id={}) for {}, size={}bytes",
+ new Object[] {Objects.hashCode(GuidePostsCache.this), key,
+ info.getEstimatedSize()});
+ }
+ }
+ }
+
+ /**
+ * Returns the underlying cache. Try to use the provided methods instead of accessing the cache
+ * directly.
+ */
+ LoadingCache<GuidePostsKey, GuidePostsInfo> getCache() {
+ return cache;
+ }
+
+ /**
+ * Returns the PTableStats for the given <code>tableName</code, using the provided
+ * <code>valueLoader</code> if no such mapping exists.
+ *
+ * @see com.google.common.cache.LoadingCache#get(Object)
+ */
+ public GuidePostsInfo get(GuidePostsKey key) throws ExecutionException {
+ return getCache().get(key);
+ }
+
+ /**
+ * Cache the given <code>stats</code> to the cache for the given <code>tableName</code>.
+ *
+ * @see com.google.common.cache.Cache#put(Object, Object)
+ */
+ public void put(GuidePostsKey key, GuidePostsInfo info) {
+ getCache().put(Objects.requireNonNull(key), Objects.requireNonNull(info));
+ }
+
+ /**
+ * Removes the mapping for <code>tableName</code> if it exists.
+ *
+ * @see com.google.common.cache.Cache#invalidate(Object)
+ */
+ public void invalidate(GuidePostsKey key) {
+ getCache().invalidate(Objects.requireNonNull(key));
+ }
+
+ /**
+ * Removes all mappings from the cache.
+ *
+ * @see com.google.common.cache.Cache#invalidateAll()
+ */
+ public void invalidateAll() {
+ getCache().invalidateAll();
+ }
+
+ /**
+ * Removes all mappings where the {@link org.apache.phoenix.schema.stats.GuidePostsKey#getPhysicalName()}
+ * equals physicalName. Because all keys in the map must be iterated, this method should be avoided.
+ * @param physicalName
+ */
+ public void invalidateAll(byte[] physicalName) {
+ for (GuidePostsKey key : getCache().asMap().keySet()) {
+ if (Bytes.compareTo(key.getPhysicalName(), physicalName) == 0) {
+ invalidate(key);
+ }
+ }
+ }
+
+ public void invalidateAll(HTableDescriptor htableDesc) {
+ byte[] tableName = htableDesc.getTableName().getName();
+ for (byte[] fam : htableDesc.getFamiliesKeys()) {
+ invalidate(new GuidePostsKey(tableName, fam));
+ }
+ }
+
+ public void invalidateAll(PTable table) {
+ byte[] physicalName = table.getPhysicalName().getBytes();
+ List<PColumnFamily> families = table.getColumnFamilies();
+ if (families.isEmpty()) {
+ invalidate(new GuidePostsKey(physicalName, SchemaUtil.getEmptyColumnFamily(table)));
+ } else {
+ for (PColumnFamily family : families) {
+ invalidate(new GuidePostsKey(physicalName, family.getName().getBytes()));
+ }
+ }
+ }
+
+ /**
+ * A {@link RemovalListener} implementation to track evictions from the table stats cache.
+ */
+ static class PhoenixStatsCacheRemovalListener implements
+ RemovalListener<GuidePostsKey, GuidePostsInfo> {
+ @Override
+ public void onRemoval(RemovalNotification<GuidePostsKey, GuidePostsInfo> notification) {
+ if (logger.isTraceEnabled()) {
+ final RemovalCause cause = notification.getCause();
+ if (wasEvicted(cause)) {
+ GuidePostsKey key = notification.getKey();
+ logger.trace("Cached stats for {} with size={}bytes was evicted due to cause={}",
+ new Object[] {key, notification.getValue().getEstimatedSize(),
+ cause});
+ }
+ }
+ }
+
+ boolean wasEvicted(RemovalCause cause) {
+ // This is actually a method on RemovalCause but isn't exposed
+ return RemovalCause.EXPLICIT != cause && RemovalCause.REPLACED != cause;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index fd37328..89f7aba 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -301,7 +301,7 @@ public interface QueryConstants {
"CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_STATS_TABLE + "\"(\n" +
// PK columns
PHYSICAL_NAME + " VARCHAR NOT NULL," +
- COLUMN_FAMILY + " VARCHAR NOT NULL," +
+ COLUMN_FAMILY + " VARCHAR," +
GUIDE_POST_KEY + " VARBINARY," +
GUIDE_POSTS_WIDTH + " BIGINT," +
LAST_STATS_UPDATE_TIME+ " DATE, "+
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java b/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java
deleted file mode 100644
index 2c7b2db..0000000
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/TableStatsCache.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.query;
-
-import java.io.IOException;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
-import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.schema.stats.PTableStats;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
-import org.apache.phoenix.util.SchemaUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.cache.RemovalCause;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.common.cache.Weigher;
-
-/**
- * "Client-side" cache for storing {@link PTableStats} for Phoenix tables. Intended to decouple
- * Phoenix from a specific version of Guava's cache.
- */
-public class TableStatsCache {
- private static final Logger logger = LoggerFactory.getLogger(TableStatsCache.class);
-
- private final ConnectionQueryServices queryServices;
- private final LoadingCache<ImmutableBytesPtr, PTableStats> cache;
-
- public TableStatsCache(ConnectionQueryServices queryServices, Configuration config) {
- this.queryServices = Objects.requireNonNull(queryServices);
- // Number of millis to expire cache values after write
- final long statsUpdateFrequency = config.getLong(
- QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS);
- // Maximum number of entries (tables) to store in the cache at one time
- final long maxTableStatsCacheSize = config.getLong(
- QueryServices.STATS_MAX_CACHE_SIZE,
- QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE);
- cache = CacheBuilder.newBuilder()
- // Expire entries a given amount of time after they were written
- .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS)
- // Maximum total weight (size in bytes) of stats entries
- .maximumWeight(maxTableStatsCacheSize)
- // Defer actual size to the PTableStats.getEstimatedSize()
- .weigher(new Weigher<ImmutableBytesPtr, PTableStats>() {
- @Override public int weigh(ImmutableBytesPtr key, PTableStats stats) {
- return stats.getEstimatedSize();
- }
- })
- // Log removals at TRACE for debugging
- .removalListener(new PhoenixStatsCacheRemovalListener())
- // Automatically load the cache when entries are missing
- .build(new StatsLoader());
- }
-
- /**
- * {@link CacheLoader} implementation for the Phoenix Table Stats cache.
- */
- protected class StatsLoader extends CacheLoader<ImmutableBytesPtr, PTableStats> {
- @Override
- public PTableStats load(ImmutableBytesPtr tableName) throws Exception {
- @SuppressWarnings("deprecation")
- HTableInterface statsHTable = queryServices.getTable(SchemaUtil.getPhysicalName(
- PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,
- queryServices.getProps()).getName());
- final byte[] tableNameBytes = tableName.copyBytesIfNecessary();
- try {
- PTableStats stats = StatisticsUtil.readStatistics(statsHTable, tableNameBytes,
- Long.MAX_VALUE);
- traceStatsUpdate(tableNameBytes, stats);
- return stats;
- } catch (TableNotFoundException e) {
- // On a fresh install, stats might not yet be created, don't warn about this.
- logger.debug("Unable to locate Phoenix stats table", e);
- return PTableStats.EMPTY_STATS;
- } catch (IOException e) {
- logger.warn("Unable to read from stats table", e);
- // Just cache empty stats. We'll try again after some time anyway.
- return PTableStats.EMPTY_STATS;
- } finally {
- try {
- statsHTable.close();
- } catch (IOException e) {
- // Log, but continue. We have our stats anyway now.
- logger.warn("Unable to close stats table", e);
- }
- }
- }
-
- /**
- * Logs a trace message for newly inserted entries to the stats cache.
- */
- void traceStatsUpdate(byte[] tableName, PTableStats stats) {
- logger.trace("Updating local TableStats cache (id={}) for {}, size={}bytes",
- new Object[] {Objects.hashCode(TableStatsCache.this), Bytes.toString(tableName),
- stats.getEstimatedSize()});
- }
- }
-
- /**
- * Returns the underlying cache. Try to use the provided methods instead of accessing the cache
- * directly.
- */
- LoadingCache<ImmutableBytesPtr, PTableStats> getCache() {
- return cache;
- }
-
- /**
- * Returns the PTableStats for the given <code>tableName</code, using the provided
- * <code>valueLoader</code> if no such mapping exists.
- *
- * @see com.google.common.cache.LoadingCache#get(Object)
- */
- public PTableStats get(ImmutableBytesPtr tableName) throws ExecutionException {
- return getCache().get(tableName);
- }
-
- /**
- * Cache the given <code>stats</code> to the cache for the given <code>tableName</code>.
- *
- * @see com.google.common.cache.Cache#put(Object, Object)
- */
- public void put(ImmutableBytesPtr tableName, PTableStats stats) {
- getCache().put(Objects.requireNonNull(tableName), Objects.requireNonNull(stats));
- }
-
- /**
- * Removes the mapping for <code>tableName</code> if it exists.
- *
- * @see com.google.common.cache.Cache#invalidate(Object)
- */
- public void invalidate(ImmutableBytesPtr tableName) {
- getCache().invalidate(Objects.requireNonNull(tableName));
- }
-
- /**
- * Removes all mappings from the cache.
- *
- * @see com.google.common.cache.Cache#invalidateAll()
- */
- public void invalidateAll() {
- getCache().invalidateAll();
- }
-
- /**
- * A {@link RemovalListener} implementation to track evictions from the table stats cache.
- */
- static class PhoenixStatsCacheRemovalListener implements
- RemovalListener<ImmutableBytesPtr, PTableStats> {
- @Override
- public void onRemoval(RemovalNotification<ImmutableBytesPtr, PTableStats> notification) {
- final RemovalCause cause = notification.getCause();
- if (wasEvicted(cause)) {
- ImmutableBytesPtr ptr = notification.getKey();
- String tableName = new String(ptr.get(), ptr.getOffset(), ptr.getLength());
- logger.trace("Cached stats for {} with size={}bytes was evicted due to cause={}",
- new Object[] {tableName, notification.getValue().getEstimatedSize(),
- cause});
- }
- }
-
- boolean wasEvicted(RemovalCause cause) {
- // This is actually a method on RemovalCause but isn't exposed
- return RemovalCause.EXPLICIT != cause && RemovalCause.REPLACED != cause;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 0c53d3e..54c6361 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -189,7 +189,7 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
-import org.apache.phoenix.schema.stats.PTableStats;
+import org.apache.phoenix.schema.stats.GuidePostsKey;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PInteger;
@@ -1085,9 +1085,20 @@ public class MetaDataClient {
* This supports scenarios in which a major compaction was manually initiated and the
* client wants the modified stats to be reflected immediately.
*/
- connection.getQueryServices().clearTableFromCache(tenantIdBytes,
- Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
- Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
+ if (cfs == null) {
+ List<PColumnFamily> families = logicalTable.getColumnFamilies();
+ if (families.isEmpty()) {
+ connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), SchemaUtil.getEmptyColumnFamily(logicalTable)));
+ } else {
+ for (PColumnFamily family : families) {
+ connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), family.getName().getBytes()));
+ }
+ }
+ } else {
+ for (byte[] cf : cfs) {
+ connection.getQueryServices().invalidateStats(new GuidePostsKey(physicalName.getBytes(), cf));
+ }
+ }
return rowCount;
}
@@ -1760,7 +1771,7 @@ public class MetaDataClient {
updateCacheFrequency = updateCacheFrequencyProp;
}
String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps);
- Long guidePostWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps);
+ Long guidePostsWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps);
Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps);
if (storeNullsProp == null) {
@@ -2227,10 +2238,10 @@ public class MetaDataClient {
tableUpsert.setString(23, autoPartitionSeq);
}
tableUpsert.setBoolean(24, isAppendOnlySchema);
- if (guidePostWidth == null) {
+ if (guidePostsWidth == null) {
tableUpsert.setNull(25, Types.BIGINT);
} else {
- tableUpsert.setLong(25, guidePostWidth);
+ tableUpsert.setLong(25, guidePostsWidth);
}
tableUpsert.execute();
@@ -3108,7 +3119,7 @@ public class MetaDataClient {
connection.setAutoCommit(true);
// Delete everything in the column. You'll still be able to do queries at earlier timestamps
long ts = (scn == null ? result.getMutationTime() : scn);
- MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, Collections.singletonList(projectCF), null, ts);
+ MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, projectCF == null ? null : Collections.singletonList(projectCF), null, ts);
return connection.getQueryServices().updateData(plan);
}
return new MutationState(0,connection);
@@ -3532,28 +3543,6 @@ public class MetaDataClient {
connection.addSchema(result.getSchema());
}
- public PTableStats getTableStats(PTable table) throws SQLException {
- /*
- * The shared view index case is tricky, because we don't have
- * table meta data for it, only an HBase table. We do have stats,
- * though, so we'll query them directly here and cache them so
- * we don't keep querying for them.
- */
- boolean isSharedIndex = table.getViewIndexId() != null;
- if (isSharedIndex) {
- // we are assuming the stats table is not transactional
- return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(),
- getCurrentScn());
- }
- boolean isView = table.getType() == PTableType.VIEW;
- String physicalName = table.getPhysicalName().toString().replace(QueryConstants.NAMESPACE_SEPARATOR,
- QueryConstants.NAME_SEPARATOR);
- if (isView && table.getViewType() != ViewType.MAPPED) {
- return connection.getQueryServices().getTableStats(Bytes.toBytes(physicalName), getCurrentScn());
- }
- return connection.getQueryServices().getTableStats(table.getName().getBytes(), getCurrentScn());
- }
-
private void throwIfLastPKOfParentIsFixedLength(PTable parent, String viewSchemaName, String viewName, ColumnDef col) throws SQLException {
if (isLastPKVariableLength(parent)) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_MODIFY_VIEW_PK)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a5b710d1/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index 0cf5ed8..9cff48c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -64,7 +65,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
private final Pair<Long, GuidePostsInfoBuilder> cachedGuidePosts;
private final byte[] guidePostWidthBytes;
private final byte[] guidePostPerRegionBytes;
- // Where to look for GUIDE_POST_WIDTH in SYSTEM.CATALOG
+ // Where to look for GUIDE_POSTS_WIDTH in SYSTEM.CATALOG
private final byte[] ptableKey;
private final RegionCoprocessorEnvironment env;
@@ -124,12 +125,12 @@ class DefaultStatisticsCollector implements StatisticsCollector {
Get get = new Get(ptableKey);
get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
Result result = htable.get(get);
- long guidepostWidth = 0;
+ long guidepostWidth = -1;
if (!result.isEmpty()) {
Cell cell = result.listCells().get(0);
guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(), cell.getValueOffset(), SortOrder.getDefault());
}
- if (guidepostWidth > 0) {
+ if (guidepostWidth >= 0) {
this.guidePostDepth = guidepostWidth;
} else {
// Last use global config value
@@ -157,10 +158,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
@Override
- public void updateStatistic(Region region) {
+ public void updateStatistic(Region region, Scan scan) {
try {
ArrayList<Mutation> mutations = new ArrayList<Mutation>();
- writeStatsToStatsTable(region, true, mutations, TimeKeeper.SYSTEM.getCurrentTime());
+ writeStatistics(region, true, mutations, TimeKeeper.SYSTEM.getCurrentTime(), scan);
if (logger.isDebugEnabled()) {
logger.debug("Committing new stats for the region " + region.getRegionInfo());
}
@@ -170,16 +171,20 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
}
- private void writeStatsToStatsTable(final Region region, boolean delete, List<Mutation> mutations, long currentTime)
+ private void writeStatistics(final Region region, boolean delete, List<Mutation> mutations, long currentTime, Scan scan)
throws IOException {
try {
Set<ImmutableBytesPtr> fams = guidePostsInfoWriterMap.keySet();
- // update the statistics table
- // Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS
- // This will not impact a stats collection of single column family during compaction as
- // guidePostsInfoWriterMap cannot be empty in this case.
+ // Update the statistics table.
+ // Delete statistics for a region if no guide posts are collected for that region during
+ // UPDATE STATISTICS. This will not impact a stats collection of single column family during
+ // compaction as guidePostsInfoWriterMap cannot be empty in this case.
if (cachedGuidePosts == null) {
- boolean collectingForLocalIndex = !fams.isEmpty() && MetaDataUtil.isLocalIndexFamily(fams.iterator().next());
+ // We're either collecting stats for the data table or the local index table, but not both
+ // We can determine this based on the column families in the scan being prefixed with the
+ // local index column family prefix. We always explicitly specify the local index column
+ // families when we're collecting stats for a local index.
+ boolean collectingForLocalIndex = scan != null && !scan.getFamilyMap().isEmpty() && MetaDataUtil.isLocalIndexFamily(scan.getFamilyMap().keySet().iterator().next());
for (Store store : region.getStores()) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
boolean isLocalIndexStore = MetaDataUtil.isLocalIndexFamily(cfKey);
@@ -202,7 +207,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
if (logger.isDebugEnabled()) {
logger.debug("Adding new stats for the region " + region.getRegionInfo());
}
- statsWriter.addStats(this, fam, mutations);
+ // If we've disabled stats, don't write any, just delete them
+ if (this.guidePostDepth > 0) {
+ statsWriter.addStats(this, fam, mutations);
+ }
}
} catch (IOException e) {
logger.error("Failed to update statistics table!", e);
@@ -223,6 +231,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
*/
@Override
public void collectStatistics(final List<Cell> results) {
+ // A guide posts depth of zero disables the collection of stats
+ if (guidePostDepth == 0) {
+ return;
+ }
Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
boolean incrementRow = true;
for (Cell cell : results) {
@@ -274,8 +286,12 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
private InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, InternalScanner internalScan,
- ImmutableBytesPtr family) {
- return new StatisticsScanner(this, statsWriter, env, internalScan, family);
+ ImmutableBytesPtr family) throws IOException {
+ StatisticsScanner scanner = new StatisticsScanner(this, statsWriter, env, internalScan, family);
+ // We need to initialize the scanner synchronously and potentially perform a cross region Get
+ // in order to use the correct guide posts width for the table being compacted.
+ init();
+ return scanner;
}
@Override