You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2018/03/14 04:37:35 UTC
[4/9] phoenix git commit: PHOENIX-4437 Make
QueryPlan.getEstimatedBytesToScan() independent of getExplainPlan() and pull
optimize() out of getExplainPlan()
PHOENIX-4437 Make QueryPlan.getEstimatedBytesToScan() independent of getExplainPlan() and pull optimize() out of getExplainPlan()
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7ef96fe1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7ef96fe1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7ef96fe1
Branch: refs/heads/4.x-HBase-0.98
Commit: 7ef96fe1bed43f3ac3dae900a3e6a83791faf697
Parents: 977699a
Author: maryannxue <ma...@gmail.com>
Authored: Thu Dec 21 10:31:04 2017 -0800
Committer: maryannxue <ma...@gmail.com>
Committed: Tue Mar 13 20:08:24 2018 -0700
----------------------------------------------------------------------
.../end2end/ExplainPlanWithStatsEnabledIT.java | 4 +-
.../apache/phoenix/execute/BaseQueryPlan.java | 45 ++++++--------
.../apache/phoenix/execute/HashJoinPlan.java | 59 +++++++++---------
.../phoenix/execute/SortMergeJoinPlan.java | 63 ++++++++++----------
.../org/apache/phoenix/execute/UnionPlan.java | 53 ++++++++--------
.../apache/phoenix/jdbc/PhoenixStatement.java | 9 ++-
6 files changed, 120 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
index 76ed7ba..a835e84 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
@@ -303,8 +303,8 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT {
try (Connection conn = DriverManager.getConnection(getUrl())) {
conn.setAutoCommit(false);
Estimate info = getByteRowEstimates(conn, sql, binds);
- assertEquals((Long) 200L, info.estimatedBytes);
- assertEquals((Long) 2L, info.estimatedRows);
+ assertEquals((Long) 176l, info.estimatedBytes);
+ assertEquals((Long) 2l, info.estimatedRows);
assertTrue(info.estimateInfoTs > 0);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 38ed926..c4edf31 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -118,7 +118,7 @@ public abstract class BaseQueryPlan implements QueryPlan {
protected Long estimatedRows;
protected Long estimatedSize;
protected Long estimateInfoTimestamp;
- private boolean explainPlanCalled;
+ private boolean getEstimatesCalled;
protected BaseQueryPlan(
@@ -500,32 +500,17 @@ public abstract class BaseQueryPlan implements QueryPlan {
@Override
public ExplainPlan getExplainPlan() throws SQLException {
- explainPlanCalled = true;
if (context.getScanRanges() == ScanRanges.NOTHING) {
return new ExplainPlan(Collections.singletonList("DEGENERATE SCAN OVER " + getTableRef().getTable().getName().getString()));
}
- // If cost-based optimizer is enabled, we need to initialize a dummy iterator to
- // get the stats for computing costs.
- boolean costBased =
- context.getConnection().getQueryServices().getConfiguration().getBoolean(
- QueryServices.COST_BASED_OPTIMIZER_ENABLED, QueryServicesOptions.DEFAULT_COST_BASED_OPTIMIZER_ENABLED);
- if (costBased) {
- ResultIterator iterator = iterator();
- iterator.close();
- }
- // Optimize here when getting explain plan, as queries don't get optimized until after compilation
- QueryPlan plan = context.getConnection().getQueryServices().getOptimizer().optimize(context.getStatement(), this);
- ExplainPlan exp = plan instanceof BaseQueryPlan ? new ExplainPlan(getPlanSteps(plan.iterator())) : plan.getExplainPlan();
- if (!costBased) { // do not override estimates if they are used for cost calculation.
- this.estimatedRows = plan.getEstimatedRowsToScan();
- this.estimatedSize = plan.getEstimatedBytesToScan();
- this.estimateInfoTimestamp = plan.getEstimateInfoTimestamp();
- }
- return exp;
+ ResultIterator iterator = iterator();
+ ExplainPlan explainPlan = new ExplainPlan(getPlanSteps(iterator));
+ iterator.close();
+ return explainPlan;
}
- private List<String> getPlanSteps(ResultIterator iterator){
+ private List<String> getPlanSteps(ResultIterator iterator) {
List<String> planSteps = Lists.newArrayListWithExpectedSize(5);
iterator.explain(planSteps);
return planSteps;
@@ -538,26 +523,32 @@ public abstract class BaseQueryPlan implements QueryPlan {
@Override
public Long getEstimatedRowsToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedRows;
}
@Override
public Long getEstimatedBytesToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedSize;
}
@Override
public Long getEstimateInfoTimestamp() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimateInfoTimestamp;
}
+ private void getEstimates() throws SQLException {
+ getEstimatesCalled = true;
+ // Initialize a dummy iterator to get the estimates based on stats.
+ ResultIterator iterator = iterator();
+ iterator.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
index 2d2ff4e..23a0da6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/HashJoinPlan.java
@@ -99,7 +99,7 @@ public class HashJoinPlan extends DelegateQueryPlan {
private Long estimatedRows;
private Long estimatedBytes;
private Long estimateInfoTs;
- private boolean explainPlanCalled;
+ private boolean getEstimatesCalled;
public static HashJoinPlan create(SelectStatement statement,
QueryPlan plan, HashJoinInfo joinInfo, SubPlan[] subPlans) throws SQLException {
@@ -247,7 +247,6 @@ public class HashJoinPlan extends DelegateQueryPlan {
@Override
public ExplainPlan getExplainPlan() throws SQLException {
- explainPlanCalled = true;
List<String> planSteps = Lists.newArrayList(delegate.getExplainPlan().getPlanSteps());
int count = subPlans.length;
for (int i = 0; i < count; i++) {
@@ -263,26 +262,6 @@ public class HashJoinPlan extends DelegateQueryPlan {
if (joinInfo != null && joinInfo.getLimit() != null) {
planSteps.add(" JOIN-SCANNER " + joinInfo.getLimit() + " ROW LIMIT");
}
- for (SubPlan subPlan : subPlans) {
- if (subPlan.getInnerPlan().getEstimatedBytesToScan() == null
- || subPlan.getInnerPlan().getEstimatedRowsToScan() == null
- || subPlan.getInnerPlan().getEstimateInfoTimestamp() == null) {
- /*
- * If any of the sub plans doesn't have the estimate info available, then we don't
- * provide estimate for the overall plan
- */
- estimatedBytes = null;
- estimatedRows = null;
- estimateInfoTs = null;
- break;
- } else {
- estimatedBytes =
- add(estimatedBytes, subPlan.getInnerPlan().getEstimatedBytesToScan());
- estimatedRows = add(estimatedRows, subPlan.getInnerPlan().getEstimatedRowsToScan());
- estimateInfoTs =
- getMin(estimateInfoTs, subPlan.getInnerPlan().getEstimateInfoTimestamp());
- }
- }
return new ExplainPlan(planSteps);
}
@@ -520,27 +499,51 @@ public class HashJoinPlan extends DelegateQueryPlan {
@Override
public Long getEstimatedRowsToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedRows;
}
@Override
public Long getEstimatedBytesToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedBytes;
}
@Override
public Long getEstimateInfoTimestamp() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimateInfoTs;
}
+
+ private void getEstimates() throws SQLException {
+ getEstimatesCalled = true;
+ for (SubPlan subPlan : subPlans) {
+ if (subPlan.getInnerPlan().getEstimatedBytesToScan() == null
+ || subPlan.getInnerPlan().getEstimatedRowsToScan() == null
+ || subPlan.getInnerPlan().getEstimateInfoTimestamp() == null) {
+ /*
+ * If any of the sub plans doesn't have the estimate info available, then we don't
+ * provide estimate for the overall plan
+ */
+ estimatedBytes = null;
+ estimatedRows = null;
+ estimateInfoTs = null;
+ break;
+ } else {
+ estimatedBytes =
+ add(estimatedBytes, subPlan.getInnerPlan().getEstimatedBytesToScan());
+ estimatedRows = add(estimatedRows, subPlan.getInnerPlan().getEstimatedRowsToScan());
+ estimateInfoTs =
+ getMin(estimateInfoTs, subPlan.getInnerPlan().getEstimateInfoTimestamp());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
index 3e380da..2436d1e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
@@ -95,7 +95,7 @@ public class SortMergeJoinPlan implements QueryPlan {
private Long estimatedBytes;
private Long estimatedRows;
private Long estimateInfoTs;
- private boolean explainPlanCalled;
+ private boolean getEstimatesCalled;
public SortMergeJoinPlan(StatementContext context, FilterableStatement statement, TableRef table,
JoinType type, QueryPlan lhsPlan, QueryPlan rhsPlan, List<Expression> lhsKeyExpressions, List<Expression> rhsKeyExpressions,
@@ -157,7 +157,6 @@ public class SortMergeJoinPlan implements QueryPlan {
@Override
public ExplainPlan getExplainPlan() throws SQLException {
- explainPlanCalled = true;
List<String> steps = Lists.newArrayList();
steps.add("SORT-MERGE-JOIN (" + type.toString().toUpperCase() + ") TABLES");
for (String step : lhsPlan.getExplainPlan().getPlanSteps()) {
@@ -167,28 +166,6 @@ public class SortMergeJoinPlan implements QueryPlan {
for (String step : rhsPlan.getExplainPlan().getPlanSteps()) {
steps.add(" " + step);
}
- if ((lhsPlan.getEstimatedBytesToScan() == null || rhsPlan.getEstimatedBytesToScan() == null)
- || (lhsPlan.getEstimatedRowsToScan() == null
- || rhsPlan.getEstimatedRowsToScan() == null)
- || (lhsPlan.getEstimateInfoTimestamp() == null
- || rhsPlan.getEstimateInfoTimestamp() == null)) {
- /*
- * If any of the sub plans doesn't have the estimate info available, then we don't
- * provide estimate for the overall plan
- */
- estimatedBytes = null;
- estimatedRows = null;
- estimateInfoTs = null;
- } else {
- estimatedBytes =
- add(add(estimatedBytes, lhsPlan.getEstimatedBytesToScan()),
- rhsPlan.getEstimatedBytesToScan());
- estimatedRows =
- add(add(estimatedRows, lhsPlan.getEstimatedRowsToScan()),
- rhsPlan.getEstimatedRowsToScan());
- estimateInfoTs =
- getMin(lhsPlan.getEstimateInfoTimestamp(), rhsPlan.getEstimateInfoTimestamp());
- }
return new ExplainPlan(steps);
}
@@ -754,25 +731,51 @@ public class SortMergeJoinPlan implements QueryPlan {
@Override
public Long getEstimatedRowsToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedRows;
}
@Override
public Long getEstimatedBytesToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedBytes;
}
@Override
public Long getEstimateInfoTimestamp() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimateInfoTs;
}
+
+ private void getEstimates() throws SQLException {
+ getEstimatesCalled = true;
+ if ((lhsPlan.getEstimatedBytesToScan() == null || rhsPlan.getEstimatedBytesToScan() == null)
+ || (lhsPlan.getEstimatedRowsToScan() == null
+ || rhsPlan.getEstimatedRowsToScan() == null)
+ || (lhsPlan.getEstimateInfoTimestamp() == null
+ || rhsPlan.getEstimateInfoTimestamp() == null)) {
+ /*
+ * If any of the sub plans doesn't have the estimate info available, then we don't
+ * provide estimate for the overall plan
+ */
+ estimatedBytes = null;
+ estimatedRows = null;
+ estimateInfoTs = null;
+ } else {
+ estimatedBytes =
+ add(add(estimatedBytes, lhsPlan.getEstimatedBytesToScan()),
+ rhsPlan.getEstimatedBytesToScan());
+ estimatedRows =
+ add(add(estimatedRows, lhsPlan.getEstimatedRowsToScan()),
+ rhsPlan.getEstimatedRowsToScan());
+ estimateInfoTs =
+ getMin(lhsPlan.getEstimateInfoTimestamp(), rhsPlan.getEstimateInfoTimestamp());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
index e6bf654..3b5168c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
@@ -69,7 +69,7 @@ public class UnionPlan implements QueryPlan {
private Long estimatedRows;
private Long estimatedBytes;
private Long estimateInfoTs;
- private boolean explainPlanCalled;
+ private boolean getEstimatesCalled;
public UnionPlan(StatementContext context, FilterableStatement statement, TableRef table, RowProjector projector,
Integer limit, Integer offset, OrderBy orderBy, GroupBy groupBy, List<QueryPlan> plans, ParameterMetaData paramMetaData) throws SQLException {
@@ -174,7 +174,6 @@ public class UnionPlan implements QueryPlan {
@Override
public ExplainPlan getExplainPlan() throws SQLException {
- explainPlanCalled = true;
List<String> steps = new ArrayList<String>();
steps.add("UNION ALL OVER " + this.plans.size() + " QUERIES");
ResultIterator iterator = iterator();
@@ -184,23 +183,6 @@ public class UnionPlan implements QueryPlan {
for (int i = 1 ; i < steps.size()-offset; i++) {
steps.set(i, " " + steps.get(i));
}
- for (QueryPlan plan : plans) {
- if (plan.getEstimatedBytesToScan() == null || plan.getEstimatedRowsToScan() == null
- || plan.getEstimateInfoTimestamp() == null) {
- /*
- * If any of the sub plans doesn't have the estimate info available, then we don't
- * provide estimate for the overall plan
- */
- estimatedBytes = null;
- estimatedRows = null;
- estimateInfoTs = null;
- break;
- } else {
- estimatedBytes = add(estimatedBytes, plan.getEstimatedBytesToScan());
- estimatedRows = add(estimatedRows, plan.getEstimatedRowsToScan());
- estimateInfoTs = getMin(estimateInfoTs, plan.getEstimateInfoTimestamp());
- }
- }
return new ExplainPlan(steps);
}
@@ -265,25 +247,46 @@ public class UnionPlan implements QueryPlan {
@Override
public Long getEstimatedRowsToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedRows;
}
@Override
public Long getEstimatedBytesToScan() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimatedBytes;
}
@Override
public Long getEstimateInfoTimestamp() throws SQLException {
- if (!explainPlanCalled) {
- getExplainPlan();
+ if (!getEstimatesCalled) {
+ getEstimates();
}
return estimateInfoTs;
}
+
+ private void getEstimates() throws SQLException {
+ getEstimatesCalled = true;
+ for (QueryPlan plan : plans) {
+ if (plan.getEstimatedBytesToScan() == null || plan.getEstimatedRowsToScan() == null
+ || plan.getEstimateInfoTimestamp() == null) {
+ /*
+ * If any of the sub plans doesn't have the estimate info available, then we don't
+ * provide estimate for the overall plan
+ */
+ estimatedBytes = null;
+ estimatedRows = null;
+ estimateInfoTs = null;
+ break;
+ } else {
+ estimatedBytes = add(estimatedBytes, plan.getEstimatedBytesToScan());
+ estimatedRows = add(estimatedRows, plan.getEstimatedRowsToScan());
+ estimateInfoTs = getMin(estimateInfoTs, plan.getEstimateInfoTimestamp());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7ef96fe1/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 94aeb34..da8beae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -578,7 +578,14 @@ public class PhoenixStatement implements Statement, SQLCloseable {
@Override
public QueryPlan compilePlan(PhoenixStatement stmt, Sequence.ValueOp seqAction) throws SQLException {
CompilableStatement compilableStmt = getStatement();
- final StatementPlan plan = compilableStmt.compilePlan(stmt, Sequence.ValueOp.VALIDATE_SEQUENCE);
+ StatementPlan compilePlan = compilableStmt.compilePlan(stmt, Sequence.ValueOp.VALIDATE_SEQUENCE);
+ // For a QueryPlan, we need to get its optimized plan; for a MutationPlan, its enclosed QueryPlan
+ // has already been optimized during compilation.
+ if (compilePlan instanceof QueryPlan) {
+ QueryPlan dataPlan = (QueryPlan) compilePlan;
+ compilePlan = stmt.getConnection().getQueryServices().getOptimizer().optimize(stmt, dataPlan);
+ }
+ final StatementPlan plan = compilePlan;
List<String> planSteps = plan.getExplainPlan().getPlanSteps();
List<Tuple> tuples = Lists.newArrayListWithExpectedSize(planSteps.size());
Long estimatedBytesToScan = plan.getEstimatedBytesToScan();