You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ap...@apache.org on 2015/05/20 19:58:16 UTC
[06/31] phoenix git commit: PHOENIX-1746 Pass through guidepost
config params on UPDATE STATISTICS call
PHOENIX-1746 Pass through guidepost config params on UPDATE STATISTICS call
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/096586e6
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/096586e6
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/096586e6
Branch: refs/heads/4.x-HBase-1.x
Commit: 096586e65e2779433bf30c30e97f78ae2316365e
Parents: e06ceaf
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Mar 23 10:28:23 2015 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Mar 23 10:29:29 2015 -0700
----------------------------------------------------------------------
.../StatsCollectorWithSplitsAndMultiCFIT.java | 6 ++++++
phoenix-core/src/main/antlr3/PhoenixSQL.g | 4 ++--
.../coprocessor/BaseScannerRegionObserver.java | 6 ++++--
.../UngroupedAggregateRegionObserver.java | 4 +++-
.../apache/phoenix/jdbc/PhoenixStatement.java | 9 +++++----
.../apache/phoenix/parse/ParseNodeFactory.java | 4 ++--
.../parse/UpdateStatisticsStatement.java | 11 +++++++++-
.../apache/phoenix/schema/MetaDataClient.java | 19 ++++++++++++++----
.../schema/stats/StatisticsCollector.java | 21 +++++++++++++++-----
9 files changed, 63 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
index c34d598..bcb3a0a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorWithSplitsAndMultiCFIT.java
@@ -136,6 +136,12 @@ public class StatsCollectorWithSplitsAndMultiCFIT extends StatsCollectorAbstract
assertRowCountAndByteCount(info, rowCountArr[i], byteCountArr[i]);
i++;
}
+
+ TestUtil.analyzeTable(conn, STATS_TEST_TABLE_NAME_NEW);
+ String query = "UPDATE STATISTICS " + STATS_TEST_TABLE_NAME_NEW + " SET \"" + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB + "\"=" + Long.toString(2000);
+ conn.createStatement().execute(query);
+ keyRanges = getAllSplits(conn, STATS_TEST_TABLE_NAME_NEW);
+ assertEquals(6, keyRanges.size());
}
protected void assertRowCountAndByteCount(GuidePostsInfo info, long rowCount, long byteCount) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/antlr3/PhoenixSQL.g
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g
index 6a2e3b9..0330a39 100644
--- a/phoenix-core/src/main/antlr3/PhoenixSQL.g
+++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g
@@ -520,8 +520,8 @@ alter_table_node returns [AlterTableStatement ret]
;
update_statistics_node returns [UpdateStatisticsStatement ret]
- : UPDATE STATISTICS t=from_table_name (s=INDEX | s=ALL | s=COLUMNS)?
- {ret = factory.updateStatistics(factory.namedTable(null, t), s == null ? StatisticsCollectionScope.getDefault() : StatisticsCollectionScope.valueOf(SchemaUtil.normalizeIdentifier(s.getText())));}
+ : UPDATE STATISTICS t=from_table_name (s=INDEX | s=ALL | s=COLUMNS)? (SET (p=properties))?
+ {ret = factory.updateStatistics(factory.namedTable(null, t), s == null ? StatisticsCollectionScope.getDefault() : StatisticsCollectionScope.valueOf(SchemaUtil.normalizeIdentifier(s.getText())), p);}
;
prop_name returns [String ret]
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index c3988a0..a2269b4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.htrace.Span;
+import org.apache.htrace.Trace;
import org.apache.phoenix.execute.TupleProjector;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.KeyValueColumnExpression;
@@ -53,8 +55,6 @@ import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.ServerUtil;
-import org.apache.htrace.Span;
-import org.apache.htrace.Trace;
import com.google.common.collect.ImmutableList;
@@ -85,6 +85,8 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
public static final String EXPECTED_UPPER_REGION_KEY = "_ExpectedUpperRegionKey";
public static final String REVERSE_SCAN = "_ReverseScan";
public static final String ANALYZE_TABLE = "_ANALYZETABLE";
+ public static final String GUIDEPOST_WIDTH_BYTES = "_GUIDEPOST_WIDTH_BYTES";
+ public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION";
/**
* Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations
* are used to augment log lines emitted by Phoenix. See https://issues.apache.org/jira/browse/PHOENIX-1198.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index fc37a84..e43e5e5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -162,8 +162,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver{
long ts = scan.getTimeRange().getMax();
StatisticsCollector stats = null;
if(ScanUtil.isAnalyzeTable(scan)) {
+ byte[] gp_width_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES);
+ byte[] gp_per_region_bytes = scan.getAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION);
// Let this throw, as this scan is being done for the sole purpose of collecting stats
- stats = new StatisticsCollector(c.getEnvironment(), region.getRegionInfo().getTable().getNameAsString(), ts);
+ stats = new StatisticsCollector(c.getEnvironment(), region.getRegionInfo().getTable().getNameAsString(), ts, gp_width_bytes, gp_per_region_bytes);
}
if (ScanUtil.isLocalIndex(scan)) {
/*
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 996d243..f802ff4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -34,6 +34,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Pair;
@@ -717,8 +718,8 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
private static class ExecutableUpdateStatisticsStatement extends UpdateStatisticsStatement implements
CompilableStatement {
- public ExecutableUpdateStatisticsStatement(NamedTableNode table, StatisticsCollectionScope scope) {
- super(table, scope);
+ public ExecutableUpdateStatisticsStatement(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) {
+ super(table, scope, props);
}
@SuppressWarnings("unchecked")
@@ -919,8 +920,8 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho
}
@Override
- public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope) {
- return new ExecutableUpdateStatisticsStatement(table, scope);
+ public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) {
+ return new ExecutableUpdateStatisticsStatement(table, scope, props);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
index 82ae821..931f327 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
@@ -357,8 +357,8 @@ public class ParseNodeFactory {
return new DivideParseNode(children);
}
- public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope) {
- return new UpdateStatisticsStatement(table, scope);
+ public UpdateStatisticsStatement updateStatistics(NamedTableNode table, StatisticsCollectionScope scope, Map<String,Object> props) {
+ return new UpdateStatisticsStatement(table, scope, props);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
index dff9f06..6f7b736 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/UpdateStatisticsStatement.java
@@ -21,6 +21,8 @@ import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.ALL;
import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.COLUMNS;
import static org.apache.phoenix.schema.stats.StatisticsCollectionScope.INDEX;
+import java.util.Map;
+
import org.apache.phoenix.schema.stats.StatisticsCollectionScope;
import com.sun.istack.NotNull;
@@ -28,9 +30,12 @@ import com.sun.istack.NotNull;
public class UpdateStatisticsStatement extends SingleTableStatement {
private final StatisticsCollectionScope scope;
- public UpdateStatisticsStatement(NamedTableNode table, @NotNull StatisticsCollectionScope scope) {
+ private final Map<String,Object> props;
+
+ public UpdateStatisticsStatement(NamedTableNode table, @NotNull StatisticsCollectionScope scope, Map<String,Object> props) {
super(table, 0);
this.scope = scope;
+ this.props = props;
}
public boolean updateColumns() {
@@ -43,5 +48,9 @@ public class UpdateStatisticsStatement extends SingleTableStatement {
public boolean updateAll() {
return scope == ALL;
+ }
+
+ public Map<String,Object> getProps() {
+ return props;
};
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 7688531..2ba0cde 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -153,6 +153,7 @@ import org.apache.phoenix.schema.PTable.LinkType;
import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarchar;
@@ -618,7 +619,7 @@ public class MetaDataClient {
PTable table = resolver.getTables().get(0).getTable();
long rowCount = 0;
if (updateStatisticsStmt.updateColumns()) {
- rowCount += updateStatisticsInternal(table.getPhysicalName(), table);
+ rowCount += updateStatisticsInternal(table.getPhysicalName(), table, updateStatisticsStmt.getProps());
}
if (updateStatisticsStmt.updateIndex()) {
// TODO: If our table is a VIEW with multiple indexes or a TABLE with local indexes,
@@ -626,7 +627,7 @@ public class MetaDataClient {
// across all indexes in that case so that we don't re-calculate the same stats
// multiple times.
for (PTable index : table.getIndexes()) {
- rowCount += updateStatisticsInternal(index.getPhysicalName(), index);
+ rowCount += updateStatisticsInternal(index.getPhysicalName(), index, updateStatisticsStmt.getProps());
}
// If analyzing the indexes of a multi-tenant table or a table with view indexes
// then analyze all of those indexes too.
@@ -654,14 +655,14 @@ public class MetaDataClient {
return PTableStats.EMPTY_STATS;
}
};
- rowCount += updateStatisticsInternal(name, indexLogicalTable);
+ rowCount += updateStatisticsInternal(name, indexLogicalTable, updateStatisticsStmt.getProps());
}
}
}
return new MutationState((int)rowCount, connection);
}
- private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException {
+ private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps) throws SQLException {
ReadOnlyProps props = connection.getQueryServices().getProps();
final long msMinBetweenUpdates = props
.getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
@@ -691,6 +692,16 @@ public class MetaDataClient {
Scan scan = plan.getContext().getScan();
scan.setCacheBlocks(false);
scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
+ if (statsProps != null) {
+ Object gp_width = statsProps.get(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB);
+ if (gp_width != null) {
+ scan.setAttribute(BaseScannerRegionObserver.GUIDEPOST_WIDTH_BYTES, PLong.INSTANCE.toBytes(gp_width));
+ }
+ Object gp_per_region = statsProps.get(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB);
+ if (gp_per_region != null) {
+ scan.setAttribute(BaseScannerRegionObserver.GUIDEPOST_PER_REGION, PInteger.INSTANCE.toBytes(gp_per_region));
+ }
+ }
MutationState mutationState = plan.execute();
rowCount = mutationState.getUpdateCount();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/096586e6/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index ade0fba..d6f25c4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -38,6 +38,9 @@ import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
@@ -65,15 +68,23 @@ public class StatisticsCollector {
private Pair<Long,GuidePostsInfo> cachedGps = null;
public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException {
- this(env, tableName, clientTimeStamp, null);
+ this(env, tableName, clientTimeStamp, null, null, null);
+ }
+
+ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
+ this(env, tableName, clientTimeStamp, null, gp_width_bytes, gp_per_region_bytes);
}
public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family) throws IOException {
+ this(env, tableName, clientTimeStamp, family, null, null);
+ }
+
+ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family, byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
Configuration config = env.getConfiguration();
- int guidepostPerRegion = config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION);
- long guidepostWidth = config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES);
+ int guidepostPerRegion = gp_per_region_bytes == null ? config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION) : PInteger.INSTANCE.getCodec().decodeInt(gp_per_region_bytes, 0, SortOrder.getDefault());
+ long guidepostWidth = gp_width_bytes == null ? config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES) : PLong.INSTANCE.getCodec().decodeInt(gp_width_bytes, 0, SortOrder.getDefault());
this.guidepostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth, env.getRegion().getTableDesc());
// Get the stats table associated with the current table on which the CP is
// triggered