You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2018/09/21 06:06:58 UTC
phoenix git commit: PHOENIX-4008 UPDATE STATISTIC should collect all
versions of cells
Repository: phoenix
Updated Branches:
refs/heads/master 91f085a90 -> 4e8c9ab0c
PHOENIX-4008 UPDATE STATISTIC should collect all versions of cells
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4e8c9ab0
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4e8c9ab0
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4e8c9ab0
Branch: refs/heads/master
Commit: 4e8c9ab0c081dbad7db50f717b1c1ef1afd79a80
Parents: 91f085a
Author: Bin Shi <bs...@bshi-ltm0.internal.salesforce.com>
Authored: Tue Sep 18 12:19:51 2018 -0700
Committer: Karan Mehta <ka...@gmail.com>
Committed: Thu Sep 20 23:06:00 2018 -0700
----------------------------------------------------------------------
.../phoenix/schema/stats/StatsCollectorIT.java | 72 ++++++++++++++------
.../apache/phoenix/schema/MetaDataClient.java | 1 +
2 files changed, 53 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e8c9ab0/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
index 3af0d09..41e39e3 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
@@ -698,6 +697,25 @@ public abstract class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT {
assertEquals("Number of expected rows in stats table after major compaction didn't match", numRows, rs.getInt(1));
}
}
+
+ private void verifyGuidePostGenerated(ConnectionQueryServices queryServices,
+ String tableName, String[] familyNames,
+ long guidePostWidth, boolean emptyGuidePostExpected) throws Exception {
+ try (Table statsHTable =
+ queryServices.getTable(
+ SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,
+ queryServices.getProps()).getName())) {
+ for (String familyName : familyNames) {
+ GuidePostsInfo gps =
+ StatisticsUtil.readStatistics(statsHTable,
+ new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes(familyName)),
+ HConstants.LATEST_TIMESTAMP);
+ assertTrue(emptyGuidePostExpected ? gps.isEmptyGuidePost() : !gps.isEmptyGuidePost());
+ assertTrue(gps.getByteCounts()[0] >= guidePostWidth);
+ assertTrue(gps.getGuidePostTimestamps()[0] > 0);
+ }
+ }
+ }
@Test
public void testEmptyGuidePostGeneratedWhenDataSizeLessThanGPWidth() throws Exception {
@@ -714,25 +732,39 @@ public abstract class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT {
conn.createStatement().execute("UPDATE STATISTICS " + tableName);
ConnectionQueryServices queryServices =
conn.unwrap(PhoenixConnection.class).getQueryServices();
- try (Table statsHTable =
- queryServices.getTable(
- SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,
- queryServices.getProps()).getName())) {
- GuidePostsInfo gps =
- StatisticsUtil.readStatistics(statsHTable,
- new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes("C1")),
- HConstants.LATEST_TIMESTAMP);
- assertTrue(gps.isEmptyGuidePost());
- assertEquals(guidePostWidth, gps.getByteCounts()[0]);
- assertTrue(gps.getGuidePostTimestamps()[0] > 0);
- gps =
- StatisticsUtil.readStatistics(statsHTable,
- new GuidePostsKey(Bytes.toBytes(tableName), Bytes.toBytes("C2")),
- HConstants.LATEST_TIMESTAMP);
- assertTrue(gps.isEmptyGuidePost());
- assertEquals(guidePostWidth, gps.getByteCounts()[0]);
- assertTrue(gps.getGuidePostTimestamps()[0] > 0);
- }
+ verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, true);
+ }
+ }
+
+ @Test
+ public void testCollectingAllVersionsOfCells() throws Exception {
+ String tableName = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ long guidePostWidth = 70;
+ String ddl =
+ "CREATE TABLE " + tableName + " (k INTEGER PRIMARY KEY, c1.a bigint, c2.b bigint)"
+ + " GUIDE_POSTS_WIDTH=" + guidePostWidth
+ + ", USE_STATS_FOR_PARALLELIZATION=true" + ", VERSIONS=3";
+ conn.createStatement().execute(ddl);
+ conn.createStatement().execute("upsert into " + tableName + " values (100,100,3)");
+ conn.commit();
+ conn.createStatement().execute("UPDATE STATISTICS " + tableName);
+
+ ConnectionQueryServices queryServices =
+ conn.unwrap(PhoenixConnection.class).getQueryServices();
+
+ // The table only has one row. All cells just has one version, and the data size of the row
+ // is less than the guide post width, so we generate empty guide post.
+ verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, true);
+
+
+ conn.createStatement().execute("upsert into " + tableName + " values (100,101,4)");
+ conn.commit();
+ conn.createStatement().execute("UPDATE STATISTICS " + tableName);
+
+ // We updated the row. Now each cell has two versions, and the data size of the row
+ // is >= the guide post width, so we generate non-empty guide post.
+ verifyGuidePostGenerated(queryServices, tableName, new String[] {"C1", "C2"}, guidePostWidth, false);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4e8c9ab0/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index cdbf234..5c79603 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -1279,6 +1279,7 @@ public class MetaDataClient {
MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, cfs, null, clientTimeStamp);
Scan scan = plan.getContext().getScan();
scan.setCacheBlocks(false);
+ scan.readAllVersions();
scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES);
boolean runUpdateStatsAsync = props.getBoolean(QueryServices.RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC);
scan.setAttribute(RUN_UPDATE_STATS_ASYNC_ATTRIB, runUpdateStatsAsync ? TRUE_BYTES : FALSE_BYTES);