You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2016/07/10 13:02:34 UTC
phoenix git commit: PHOENIX-3040 Don't use guideposts for executing
queries serially simplification
Repository: phoenix
Updated Branches:
refs/heads/master dca188f02 -> b19aba2bb
PHOENIX-3040 Don't use guideposts for executing queries serially simplification
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b19aba2b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b19aba2b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b19aba2b
Branch: refs/heads/master
Commit: b19aba2bb11f3b9c94ca0a593159828ee029e2c8
Parents: dca188f
Author: James Taylor <ja...@apache.org>
Authored: Sun Jul 10 10:43:25 2016 +0200
Committer: James Taylor <ja...@apache.org>
Committed: Sun Jul 10 15:02:15 2016 +0200
----------------------------------------------------------------------
.../phoenix/compile/ListJarsQueryPlan.java | 5 --
.../org/apache/phoenix/compile/QueryPlan.java | 2 -
.../apache/phoenix/compile/TraceQueryPlan.java | 5 --
.../apache/phoenix/execute/AggregatePlan.java | 5 --
.../phoenix/execute/DelegateQueryPlan.java | 5 --
.../execute/LiteralResultIterationPlan.java | 6 ---
.../org/apache/phoenix/execute/ScanPlan.java | 48 +++-----------------
.../phoenix/execute/SortMergeJoinPlan.java | 6 ---
.../org/apache/phoenix/execute/UnionPlan.java | 9 ----
.../phoenix/iterate/BaseResultIterators.java | 9 ++--
.../apache/phoenix/iterate/SerialIterators.java | 8 ++++
.../apache/phoenix/jdbc/PhoenixStatement.java | 5 --
.../org/apache/phoenix/query/QueryServices.java | 3 +-
.../phoenix/query/QueryServicesOptions.java | 2 +-
.../query/ParallelIteratorsSplitTest.java | 5 --
15 files changed, 21 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
index e3025cc..b52e704 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ListJarsQueryPlan.java
@@ -251,9 +251,4 @@ public class ListJarsQueryPlan implements QueryPlan {
public Operation getOperation() {
return stmt.getUpdateOperation();
}
-
- @Override
- public boolean isSerial() {
- return true;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java
index f403e34..ea77d79 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/QueryPlan.java
@@ -86,6 +86,4 @@ public interface QueryPlan extends StatementPlan {
* @throws SQLException
*/
public boolean useRoundRobinIterator() throws SQLException;
-
- public boolean isSerial();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
index ed11b46..5e0977b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java
@@ -259,9 +259,4 @@ public class TraceQueryPlan implements QueryPlan {
public boolean useRoundRobinIterator() {
return false;
}
-
- @Override
- public boolean isSerial() {
- return true;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
index c439618..136379d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/AggregatePlan.java
@@ -265,9 +265,4 @@ public class AggregatePlan extends BaseQueryPlan {
public boolean useRoundRobinIterator() throws SQLException {
return false;
}
-
- @Override
- public boolean isSerial() {
- return isSerial;
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
index f282aea..8f0d224 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java
@@ -124,9 +124,4 @@ public abstract class DelegateQueryPlan implements QueryPlan {
public Integer getOffset() {
return delegate.getOffset();
}
-
- @Override
- public boolean isSerial() {
- return delegate.isSerial();
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java
index 38cb65e..7b95cf9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/LiteralResultIterationPlan.java
@@ -110,10 +110,4 @@ public class LiteralResultIterationPlan extends BaseQueryPlan {
return scanner;
}
-
- @Override
- public boolean isSerial() {
- return true;
- }
-
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
index 34354f3..7f735b7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java
@@ -60,8 +60,6 @@ import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.TableRef;
-import org.apache.phoenix.schema.stats.GuidePostsInfo;
-import org.apache.phoenix.schema.stats.PTableStats;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
@@ -135,41 +133,15 @@ public class ScanPlan extends BaseQueryPlan {
* the amount of data we need to scan is less than the threshold.
*/
return false;
- } else if (perScanLimit != null && scan.getFilter() == null) {
- /*
- * In presence of a limit and in absence of a filter, we are not relying on guide post info to
- * see if we are beyond a threshold.
- */
- float factor =
- services.getProps().getFloat(QueryServices.NONFILTERED_AND_LIMITED_QUERY_SERIAL_THRESHOLD,
- QueryServicesOptions.DEFAULT_NONFILTERED_LIMITED_QUERY_SERIAL_THRESHOLD);
- return Float.compare(estRowSize * perScanLimit, factor * regionSize) < 0;
- }
- long scn = context.getConnection().getSCN() == null ? Long.MAX_VALUE : context.getConnection().getSCN();
- PTableStats tableStats = context.getConnection().getQueryServices().getTableStats(table.getName().getBytes(), scn);
- GuidePostsInfo gpsInfo = tableStats.getGuidePosts().get(SchemaUtil.getEmptyColumnFamily(table));
- long threshold;
- if (gpsInfo == null || gpsInfo.getGuidePostsCount() == 0) {
- threshold = regionSize;
- } else {
- long totByteSize = 0;
- long totRowCount = 0;
- for (long byteCount : gpsInfo.getByteCounts()) {
- totByteSize += byteCount;
- }
- for (long rowCount : gpsInfo.getRowCounts()) {
- totRowCount += rowCount;
- }
- estRowSize = totByteSize / totRowCount;
- threshold = 2
- * services.getProps().getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES);
- }
- long thresholdToUse = services.getProps().getLong(QueryServices.FILTERED_OR_NONLIMITED_QUERY_SERIAL_THRESHOLD,
- threshold);
- return (perScanLimit * estRowSize < thresholdToUse);
+ }
+ float factor =
+ services.getProps().getFloat(QueryServices.LIMITED_QUERY_SERIAL_THRESHOLD,
+ QueryServicesOptions.DEFAULT_LIMITED_QUERY_SERIAL_THRESHOLD);
+ long threshold = (long)(factor * regionSize);
+ return (perScanLimit * estRowSize < threshold);
}
+ @SuppressWarnings("deprecation")
private static ParallelIteratorFactory buildResultIteratorFactory(StatementContext context, FilterableStatement statement,
TableRef tableRef, OrderBy orderBy, Integer limit,Integer offset, boolean allowPageFilter) throws SQLException {
@@ -292,10 +264,4 @@ public class ScanPlan extends BaseQueryPlan {
public boolean useRoundRobinIterator() throws SQLException {
return ScanUtil.isRoundRobinPossible(orderBy, context);
}
-
- @Override
- public boolean isSerial() {
- return isSerial;
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
index a8ad3eb..f4ff289 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java
@@ -671,10 +671,4 @@ public class SortMergeJoinPlan implements QueryPlan {
public Set<TableRef> getSourceRefs() {
return tableRefs;
}
-
- @Override
- public boolean isSerial() {
- return false;
- }
-
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
index f60f81f..cf95b5b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnionPlan.java
@@ -231,13 +231,4 @@ public class UnionPlan implements QueryPlan {
}
return sources;
}
-
- @Override
- public boolean isSerial() {
- boolean isSerial = true;
- for (QueryPlan plan : getPlans()) {
- isSerial &= plan.isSerial();
- }
- return isSerial;
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 523b419..6f053bf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -76,7 +76,6 @@ import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
-import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
@@ -141,14 +140,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
return plan.getTableRef().getTable();
}
- private boolean useStats() {
+ protected boolean useStats() {
/*
* Don't use guide posts:
* 1) If we're collecting stats, as in this case we need to scan entire
* regions worth of data to track where to put the guide posts.
* 2) If the query is going to be executed serially.
*/
- if (ScanUtil.isAnalyzeTable(scan) || plan.isSerial()) {
+ if (ScanUtil.isAnalyzeTable(scan)) {
return false;
}
return true;
@@ -664,8 +663,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
regionIndex++;
}
if (scanRanges.isPointLookup()) {
- this.estimatedRows = 1L;
- this.estimatedSize = SchemaUtil.estimateRowSize(table);
+ this.estimatedRows = Long.valueOf(scanRanges.getPointLookupCount());
+ this.estimatedSize = this.estimatedRows * SchemaUtil.estimateRowSize(table);
} else if (hasGuidePosts) {
this.estimatedRows = estimatedRows;
this.estimatedSize = estimatedSize;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
index 0d0c5d3..d8f7f40 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/SerialIterators.java
@@ -114,6 +114,14 @@ public class SerialIterators extends BaseResultIterators {
}
}
+ /**
+ * No need to use stats when executing serially
+ */
+ @Override
+ protected boolean useStats() {
+ return false;
+ }
+
@Override
protected String getName() {
return NAME;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 02303d1..0c154e2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -573,11 +573,6 @@ public class PhoenixStatement implements Statement, SQLCloseable {
public boolean useRoundRobinIterator() throws SQLException {
return false;
}
-
- @Override
- public boolean isSerial() {
- return true;
- }
};
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index c2acec1..cf8cb57 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -217,8 +217,7 @@ public interface QueryServices extends SQLCloseable {
// time to wait before running second index population upsert select (so that any pending batches of rows on region server are also written to index)
public static final String INDEX_POPULATION_SLEEP_TIME = "phoenix.index.population.wait.time";
public static final String LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB = "phoenix.client.localIndexUpgrade";
- public static final String NONFILTERED_AND_LIMITED_QUERY_SERIAL_THRESHOLD = "phoenix.nonfiltered.limited.query.serial.threshold";
- public static final String FILTERED_OR_NONLIMITED_QUERY_SERIAL_THRESHOLD = "phoenix.filtered.limited.query.serial.threshold";
+ public static final String LIMITED_QUERY_SERIAL_THRESHOLD = "phoenix.limited.query.serial.threshold";
/**
* Get executor service used for parallel scans
*/
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 57b60c0..64b08c1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -248,7 +248,7 @@ public class QueryServicesOptions {
(3 * DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD) / 4;
public static final int DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE = 10;
public static final boolean DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE = true;
- public static final float DEFAULT_NONFILTERED_LIMITED_QUERY_SERIAL_THRESHOLD = 0.2f;
+ public static final float DEFAULT_LIMITED_QUERY_SERIAL_THRESHOLD = 0.2f;
@SuppressWarnings("serial")
public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b19aba2b/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java
index 8c65937..05fbf81 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java
@@ -471,11 +471,6 @@ public class ParallelIteratorsSplitTest extends BaseConnectionlessQueryTest {
public boolean useRoundRobinIterator() {
return false;
}
-
- @Override
- public boolean isSerial() {
- return true;
- }
}, null, new SpoolingResultIterator.SpoolingResultIteratorFactory(context.getConnection().getQueryServices()), context.getScan(), false);
List<KeyRange> keyRanges = parallelIterators.getSplits();