You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by vj...@apache.org on 2023/07/05 22:57:29 UTC
[phoenix] branch master updated: PHOENIX-6907 Explain Plan should output region locations with servers (#1598)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new dc7d628f51 PHOENIX-6907 Explain Plan should output region locations with servers (#1598)
dc7d628f51 is described below
commit dc7d628f5194cae47228a31dfe11ec7bb38c7b46
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Wed Jul 5 15:57:23 2023 -0700
PHOENIX-6907 Explain Plan should output region locations with servers (#1598)
---
.../org/apache/phoenix/end2end/DerivedTableIT.java | 28 +++-
.../phoenix/end2end/FlappingLocalIndexIT.java | 13 ++
.../org/apache/phoenix/end2end/QueryLoggerIT.java | 4 +-
.../phoenix/end2end/index/BaseLocalIndexIT.java | 1 +
phoenix-core/src/main/antlr3/PhoenixSQL.g | 11 +-
.../phoenix/compile/ExplainPlanAttributes.java | 22 ++-
.../phoenix/iterate/BaseResultIterators.java | 3 +-
.../org/apache/phoenix/iterate/ExplainTable.java | 183 ++++++++++++++++++++-
.../org/apache/phoenix/jdbc/PhoenixStatement.java | 17 +-
.../org/apache/phoenix/parse/ExplainStatement.java | 10 +-
.../java/org/apache/phoenix/parse/ExplainType.java | 26 +++
.../org/apache/phoenix/parse/ParseNodeFactory.java | 4 +-
.../org/apache/phoenix/query/QueryServices.java | 8 +
.../apache/phoenix/query/QueryServicesOptions.java | 6 +-
14 files changed, 309 insertions(+), 27 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
index 2b47e01972..597e0fc6cb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DerivedTableIT.java
@@ -58,6 +58,8 @@ import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Category(ParallelStatsDisabledTest.class)
@@ -71,6 +73,8 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
private String[] plans;
private String tableName;
+ private static final Logger LOGGER = LoggerFactory.getLogger(DerivedTableIT.class);
+
public DerivedTableIT(String[] indexDDL, String[] plans) {
this.indexDDL = indexDDL;
@@ -114,7 +118,7 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
{
"CREATE INDEX "+dynamicTableName+"_DERIVED_IDX ON "+dynamicTableName+" (a_byte) INCLUDE (A_STRING, B_STRING)"
}, {
- "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" +
+ "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" +
" SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" +
"CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [\"B_STRING\"]\n" +
@@ -122,7 +126,7 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
"CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
"CLIENT SORTED BY [A DESC]",
- "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX\n" +
+ "CLIENT PARALLEL 1-WAY FULL SCAN OVER "+dynamicTableName+"_DERIVED_IDX \n" +
" SERVER AGGREGATE INTO DISTINCT ROWS BY [\"A_STRING\", \"B_STRING\"]\n" +
"CLIENT MERGE SORT\n" +
"CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
@@ -130,7 +134,7 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
"CLIENT SORTED BY [A DESC]"}});
testCases.add(new String[][] {
{}, {
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" +
+ "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" +
" SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" +
"CLIENT MERGE SORT\n" +
"CLIENT SORTED BY [B_STRING]\n" +
@@ -138,7 +142,7 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
"CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
"CLIENT SORTED BY [A DESC]",
- "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+"\n" +
+ "CLIENT PARALLEL 4-WAY FULL SCAN OVER "+dynamicTableName+" \n" +
" SERVER AGGREGATE INTO DISTINCT ROWS BY [A_STRING, B_STRING]\n" +
"CLIENT MERGE SORT\n" +
"CLIENT AGGREGATE INTO DISTINCT ROWS BY [A]\n" +
@@ -378,8 +382,12 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
assertFalse(rs.next());
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(plans[0], QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query);
+ String explainPlanOutput = QueryUtil.getExplainPlan(rs);
+ LOGGER.info("Explain plan output: {}", explainPlanOutput);
+ String[] splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region=");
+ String[] secondSplitExplainPlan = splitExplainPlan[1].split("]\\)");
+ assertEquals(plans[0], splitExplainPlan[0] + secondSplitExplainPlan[1]);
// distinct b (groupby a, b) groupby a orderby a
query = "SELECT DISTINCT COLLECTDISTINCT(t.b) FROM (SELECT b_string b, a_string a FROM "+tableName+" GROUP BY a_string, b_string) AS t GROUP BY t.a ORDER BY t.a DESC";
@@ -400,8 +408,12 @@ public class DerivedTableIT extends ParallelStatsDisabledIT {
assertFalse(rs.next());
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(plans[1], QueryUtil.getExplainPlan(rs));
+ rs = conn.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query);
+ explainPlanOutput = QueryUtil.getExplainPlan(rs);
+ LOGGER.info("Explain plan output: {}", explainPlanOutput);
+ splitExplainPlan = explainPlanOutput.split("\\n \\(region locations = \\[region=");
+ secondSplitExplainPlan = splitExplainPlan[1].split("]\\)");
+ assertEquals(plans[1], splitExplainPlan[0] + secondSplitExplainPlan[1]);
// (orderby) groupby
query = "SELECT t.a_string, count(*) FROM (SELECT * FROM "+tableName+" order by a_integer) AS t where a_byte != 8 group by t.a_string";
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java
index 3c76cf3cef..096a301ff7 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingLocalIndexIT.java
@@ -50,12 +50,17 @@ import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class FlappingLocalIndexIT extends BaseLocalIndexIT {
+ private static final Logger LOGGER = LoggerFactory.getLogger(FlappingLocalIndexIT.class);
+
public FlappingLocalIndexIT(boolean isNamespaceMapped) {
super(isNamespaceMapped);
}
@@ -159,6 +164,13 @@ public class FlappingLocalIndexIT extends BaseLocalIndexIT {
String query = "SELECT * FROM " + tableName +" where v1 like 'a%'";
+ String explainPlanOutput =
+ QueryUtil.getExplainPlan(conn1.createStatement().executeQuery("EXPLAIN WITH REGIONS " + query));
+ LOGGER.info("Explain plan output: {}", explainPlanOutput);
+ // MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN is set as 2
+ assertTrue("Expected total " + numRegions + " regions",
+ explainPlanOutput.contains("...total size = " + numRegions));
+
ExplainPlan plan = conn1.prepareStatement(query)
.unwrap(PhoenixPreparedStatement.class).optimizeQuery()
.getExplainPlan();
@@ -176,6 +188,7 @@ public class FlappingLocalIndexIT extends BaseLocalIndexIT {
explainPlanAttributes.getServerWhereFilter());
assertEquals("CLIENT MERGE SORT",
explainPlanAttributes.getClientSortAlgo());
+ assertEquals(numRegions, explainPlanAttributes.getRegionLocations().size());
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java
index ad8e969e80..7f78030597 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryLoggerIT.java
@@ -120,7 +120,7 @@ public class QueryLoggerIT extends BaseTest {
// sleep for sometime to let query log committed
Thread.sleep(delay);
- try (ResultSet explainRS = conn.createStatement().executeQuery("Explain " + query);
+ try (ResultSet explainRS = conn.createStatement().executeQuery("Explain with regions " + query);
ResultSet rs = conn.createStatement().executeQuery(logQuery)) {
boolean foundQueryLog = false;
@@ -300,7 +300,7 @@ public class QueryLoggerIT extends BaseTest {
// sleep for sometime to let query log committed
Thread.sleep(delay);
- String explainQuery = "Explain " + "SELECT * FROM " + tableName + " where V = 'value5'";
+ String explainQuery = "EXPLAIN WITH REGIONS " + "SELECT * FROM " + tableName + " where V = 'value5'";
try (ResultSet explainRS = conn.createStatement()
.executeQuery(explainQuery);
ResultSet rs = conn.createStatement().executeQuery(logQuery)) {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java
index 4c8863fbec..ec297c1595 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseLocalIndexIT.java
@@ -61,6 +61,7 @@ public abstract class BaseLocalIndexIT extends BaseTest {
// setting update frequency to a large value to test out that we are
// generating stats for local indexes
clientProps.put(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, "120000");
+ clientProps.put(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN, "2");
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
}
diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g
index 5d57c3669f..a8cbb71085 100644
--- a/phoenix-core/src/main/antlr3/PhoenixSQL.g
+++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g
@@ -152,6 +152,7 @@ tokens
REVOKE = 'revoke';
SHOW = 'show';
UNCOVERED = 'uncovered';
+ REGIONS = 'regions';
}
@@ -212,6 +213,7 @@ import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.parse.LikeParseNode.LikeType;
import org.apache.phoenix.trace.util.Tracing;
import org.apache.phoenix.parse.AddJarsStatement;
+import org.apache.phoenix.parse.ExplainType;
}
@lexer::header {
@@ -452,7 +454,14 @@ oneStatement returns [BindableStatement ret]
finally{ contextStack.pop(); }
explain_node returns [BindableStatement ret]
- : EXPLAIN q=oneStatement {$ret=factory.explain(q);}
+ : EXPLAIN (w=WITH)? (r=REGIONS)? q=oneStatement
+ {
+ if ((w==null && r!=null) || (w!=null && r==null)) {
+ throw new RuntimeException("Valid usage: EXPLAIN {query} OR EXPLAIN WITH REGIONS {query}");
+ }
+ ret = (w==null && r==null) ? factory.explain(q, ExplainType.DEFAULT)
+ : factory.explain(q, ExplainType.WITH_REGIONS);
+ }
;
// Parse a create table statement.
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java
index 258a7ee6b3..a41f776345 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java
@@ -18,8 +18,10 @@
package org.apache.phoenix.compile;
+import java.util.List;
import java.util.Set;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.client.Consistency;
import org.apache.phoenix.parse.HintNode;
import org.apache.phoenix.parse.HintNode.Hint;
@@ -73,6 +75,7 @@ public class ExplainPlanAttributes {
// be null
private final ExplainPlanAttributes rhsJoinQueryExplainPlan;
private final Set<PColumn> serverMergeColumns;
+ private final List<HRegionLocation> regionLocations;
private static final ExplainPlanAttributes EXPLAIN_PLAN_INSTANCE =
new ExplainPlanAttributes();
@@ -112,6 +115,7 @@ public class ExplainPlanAttributes {
this.clientSortAlgo = null;
this.rhsJoinQueryExplainPlan = null;
this.serverMergeColumns = null;
+ this.regionLocations = null;
}
public ExplainPlanAttributes(String abstractExplainPlan,
@@ -132,7 +136,7 @@ public class ExplainPlanAttributes {
Integer clientSequenceCount, String clientCursorName,
String clientSortAlgo,
ExplainPlanAttributes rhsJoinQueryExplainPlan,
- Set<PColumn> serverMergeColumns) {
+ Set<PColumn> serverMergeColumns, List<HRegionLocation> regionLocations) {
this.abstractExplainPlan = abstractExplainPlan;
this.splitsChunk = splitsChunk;
this.estimatedRows = estimatedRows;
@@ -167,6 +171,7 @@ public class ExplainPlanAttributes {
this.clientSortAlgo = clientSortAlgo;
this.rhsJoinQueryExplainPlan = rhsJoinQueryExplainPlan;
this.serverMergeColumns = serverMergeColumns;
+ this.regionLocations = regionLocations;
}
public String getAbstractExplainPlan() {
@@ -305,6 +310,10 @@ public class ExplainPlanAttributes {
return serverMergeColumns;
}
+ public List<HRegionLocation> getRegionLocations() {
+ return regionLocations;
+ }
+
public static ExplainPlanAttributes getDefaultExplainPlan() {
return EXPLAIN_PLAN_INSTANCE;
}
@@ -344,6 +353,7 @@ public class ExplainPlanAttributes {
private String clientSortAlgo;
private ExplainPlanAttributes rhsJoinQueryExplainPlan;
private Set<PColumn> serverMergeColumns;
+ private List<HRegionLocation> regionLocations;
public ExplainPlanAttributesBuilder() {
// default
@@ -396,6 +406,7 @@ public class ExplainPlanAttributes {
this.rhsJoinQueryExplainPlan =
explainPlanAttributes.getRhsJoinQueryExplainPlan();
this.serverMergeColumns = explainPlanAttributes.getServerMergeColumns();
+ this.regionLocations = explainPlanAttributes.getRegionLocations();
}
public ExplainPlanAttributesBuilder setAbstractExplainPlan(
@@ -599,6 +610,12 @@ public class ExplainPlanAttributes {
return this;
}
+ public ExplainPlanAttributesBuilder setRegionLocations(
+ List<HRegionLocation> regionLocations) {
+ this.regionLocations = regionLocations;
+ return this;
+ }
+
public ExplainPlanAttributes build() {
return new ExplainPlanAttributes(abstractExplainPlan, splitsChunk,
estimatedRows, estimatedSizeInBytes, iteratorTypeAndScanSize,
@@ -611,7 +628,8 @@ public class ExplainPlanAttributes {
clientFilterBy, clientAggregate, clientSortedBy,
clientAfterAggregate, clientDistinctFilter, clientOffset,
clientRowLimit, clientSequenceCount, clientCursorName,
- clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns);
+ clientSortAlgo, rhsJoinQueryExplainPlan, serverMergeColumns,
+ regionLocations);
}
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 08a946d085..2c6885bbf5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -1537,7 +1537,6 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
@Override
public void close() throws SQLException {
-
// Don't call cancel on already started work, as it causes the HConnection
// to get into a funk. Instead, just cancel queued work.
boolean cancelledWork = false;
@@ -1724,7 +1723,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
}
- explain(buf.toString(), planSteps, explainPlanAttributesBuilder);
+ explain(buf.toString(), planSteps, explainPlanAttributesBuilder, scans);
}
@Override
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
index c9ef46d00a..23b7632cb5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
@@ -17,15 +17,23 @@
*/
package org.apache.phoenix.iterate;
+import java.io.IOException;
+import java.sql.SQLException;
import java.text.Format;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.client.Consistency;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.filter.PageFilter;
@@ -46,6 +54,8 @@ import org.apache.phoenix.parse.HintNode;
import org.apache.phoenix.parse.HintNode.Hint;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.KeyRange.Bound;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SortOrder;
@@ -55,11 +65,17 @@ import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.StringUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public abstract class ExplainTable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ExplainTable.class);
private static final List<KeyRange> EVERYTHING = Collections.singletonList(KeyRange.EVERYTHING_RANGE);
public static final String POINT_LOOKUP_ON_STRING = "POINT LOOKUP ON ";
+ public static final String REGION_LOCATIONS = " (region locations = ";
+
protected final StatementContext context;
protected final TableRef tableRef;
protected final GroupBy groupBy;
@@ -67,7 +83,7 @@ public abstract class ExplainTable {
protected final HintNode hint;
protected final Integer limit;
protected final Integer offset;
-
+
public ExplainTable(StatementContext context, TableRef table) {
this(context, table, GroupBy.EMPTY_GROUP_BY, OrderBy.EMPTY_ORDER_BY, HintNode.EMPTY_HINT_NODE, null, null);
}
@@ -111,8 +127,65 @@ public abstract class ExplainTable {
return buf.toString();
}
- protected void explain(String prefix, List<String> planSteps,
- ExplainPlanAttributesBuilder explainPlanAttributesBuilder) {
+ /**
+ * Get regions that represent the given range of start and end key for the given table, and
+ * all the regions to the regionLocations list.
+ *
+ * @param tableName the table name.
+ * @param startKey the start rowkey.
+ * @param endKey the end rowkey.
+ * @param includeEndKey true if end key needs to be included.
+ * @param reload true if reload from meta is necessary.
+ * @param regionBoundaries set of region boundaries to get the unique list of region locations.
+ * @param regionLocations the list of region locations as output.
+ * @throws IOException if something goes wrong while creating connection or querying region
+ * locations.
+ */
+ private void getRegionsInRange(final byte[] tableName,
+ final byte[] startKey,
+ final byte[] endKey,
+ final boolean includeEndKey,
+ final boolean reload,
+ Set<RegionBoundary> regionBoundaries,
+ List<HRegionLocation> regionLocations)
+ throws IOException, SQLException {
+ final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW);
+ if ((Bytes.compareTo(startKey, endKey) > 0) && !endKeyIsEndOfTable) {
+ throw new IllegalArgumentException(
+ "Invalid range: " + Bytes.toStringBinary(startKey) + " > " +
+ Bytes.toStringBinary(endKey));
+ }
+ byte[] currentKey = startKey;
+ try (Table table = context.getConnection().getQueryServices().getTable(tableName)) {
+ // include all regions that include key range from the given start key
+ // and end key
+ do {
+ HRegionLocation regionLocation =
+ table.getRegionLocator().getRegionLocation(currentKey, reload);
+ RegionBoundary regionBoundary =
+ new RegionBoundary(regionLocation.getRegion().getStartKey(),
+ regionLocation.getRegion().getEndKey());
+ if (!regionBoundaries.contains(regionBoundary)) {
+ regionLocations.add(regionLocation);
+ regionBoundaries.add(regionBoundary);
+ }
+ currentKey = regionLocation.getRegion().getEndKey();
+ // condition1 = currentKey != END_ROW_KEY
+ // condition2 = endKeyIsEndOfTable == true
+ // condition3 = currentKey < endKey
+ // condition4 = includeEndKey == true
+ // condition5 = currentKey == endKey
+ // while (condition1 && (condition2 || condition3 || (condition4 && condition5)))
+ } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
+ && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0
+ || (includeEndKey && Bytes.compareTo(currentKey, endKey) == 0)));
+ }
+ }
+
+ protected void explain(String prefix,
+ List<String> planSteps,
+ ExplainPlanAttributesBuilder explainPlanAttributesBuilder,
+ List<List<Scan>> scansList) {
StringBuilder buf = new StringBuilder(prefix);
ScanRanges scanRanges = context.getScanRanges();
Scan scan = context.getScan();
@@ -275,6 +348,7 @@ public abstract class ExplainTable {
if (groupByLimitBytes != null) {
groupByLimit = (Integer) PInteger.INSTANCE.toObject(groupByLimitBytes);
}
+ getRegionLocations(planSteps, explainPlanAttributesBuilder, scansList);
groupBy.explain(planSteps, groupByLimit, explainPlanAttributesBuilder);
if (scan.getAttribute(BaseScannerRegionObserver.SPECIFIC_ARRAY_INDEX) != null) {
planSteps.add(" SERVER ARRAY ELEMENT PROJECTION");
@@ -284,6 +358,108 @@ public abstract class ExplainTable {
}
}
+ /**
+ * Retrieve region locations and set the values in the explain plan output.
+ *
+ * @param planSteps list of plan steps to add explain plan output to.
+ * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance.
+ * @param scansList list of the list of scans, to be used for parallel scans.
+ */
+ private void getRegionLocations(List<String> planSteps,
+ ExplainPlanAttributesBuilder explainPlanAttributesBuilder,
+ List<List<Scan>> scansList) {
+ String regionLocationPlan = getRegionLocationsForExplainPlan(explainPlanAttributesBuilder,
+ scansList);
+ if (regionLocationPlan.length() > 0) {
+ planSteps.add(regionLocationPlan);
+ }
+ }
+
+ /**
+ * Retrieve region locations from hbase client and set the values for the explain plan output.
+ * If the list of region locations exceed max limit, print only list with the max limit and
+ * print num of total list size.
+ *
+ * @param explainPlanAttributesBuilder explain plan v2 attributes builder instance.
+ * @param scansList list of the list of scans, to be used for parallel scans.
+ * @return region locations to be added to the explain plan output.
+ */
+ private String getRegionLocationsForExplainPlan(
+ ExplainPlanAttributesBuilder explainPlanAttributesBuilder,
+ List<List<Scan>> scansList) {
+ try {
+ StringBuilder buf = new StringBuilder().append(REGION_LOCATIONS);
+ Set<RegionBoundary> regionBoundaries = new HashSet<>();
+ List<HRegionLocation> regionLocations = new ArrayList<>();
+ for (List<Scan> scans : scansList) {
+ for (Scan eachScan : scans) {
+ getRegionsInRange(tableRef.getTable().getPhysicalName().getBytes(),
+ eachScan.getStartRow(),
+ eachScan.getStopRow(),
+ true,
+ false,
+ regionBoundaries,
+ regionLocations);
+ }
+ }
+ int maxLimitRegionLoc = context.getConnection().getQueryServices().getConfiguration()
+ .getInt(QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN,
+ QueryServicesOptions.DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN);
+ if (explainPlanAttributesBuilder != null) {
+ explainPlanAttributesBuilder.setRegionLocations(
+ Collections.unmodifiableList(regionLocations));
+ }
+ if (regionLocations.size() > maxLimitRegionLoc) {
+ int originalSize = regionLocations.size();
+ List<HRegionLocation> trimmedRegionLocations =
+ regionLocations.subList(0, maxLimitRegionLoc);
+ buf.append(trimmedRegionLocations);
+ buf.append("...total size = ");
+ buf.append(originalSize);
+ } else {
+ buf.append(regionLocations);
+ }
+ buf.append(") ");
+ return buf.toString();
+ } catch (IOException | SQLException | UnsupportedOperationException e) {
+ LOGGER.error("Explain table unable to add region locations.", e);
+ return "";
+ }
+ }
+
+ /**
+ * Region boundary class with start and end key of the region.
+ */
+ private static class RegionBoundary {
+ private final byte[] startKey;
+ private final byte[] endKey;
+
+ RegionBoundary(byte[] startKey, byte[] endKey) {
+ this.startKey = startKey;
+ this.endKey = endKey;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ RegionBoundary that = (RegionBoundary) o;
+ return Bytes.compareTo(startKey, that.startKey) == 0
+ && Bytes.compareTo(endKey, that.endKey) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode(startKey);
+ result = 31 * result + Arrays.hashCode(endKey);
+ return result;
+ }
+ }
+
private void appendPKColumnValue(StringBuilder buf, byte[] range, Boolean isNull, int slotIndex, boolean changeViewIndexId) {
if (Boolean.TRUE.equals(isNull)) {
buf.append("null");
@@ -414,4 +590,5 @@ public abstract class ExplainTable {
buf.setCharAt(buf.length()-1, ']');
return buf.toString();
}
+
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 44e2e98dcc..5b019d4cc4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -109,6 +109,7 @@ import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.execute.visitor.QueryPlanVisitor;
import org.apache.phoenix.expression.KeyValueColumnExpression;
import org.apache.phoenix.expression.RowKeyColumnExpression;
+import org.apache.phoenix.iterate.ExplainTable;
import org.apache.phoenix.iterate.MaterializedResultIterator;
import org.apache.phoenix.iterate.ParallelScanGrouper;
import org.apache.phoenix.iterate.ResultIterator;
@@ -140,6 +141,7 @@ import org.apache.phoenix.parse.DMLStatement;
import org.apache.phoenix.parse.DeclareCursorStatement;
import org.apache.phoenix.parse.DeleteJarStatement;
import org.apache.phoenix.parse.DeleteStatement;
+import org.apache.phoenix.parse.ExplainType;
import org.apache.phoenix.parse.ShowCreateTableStatement;
import org.apache.phoenix.parse.ShowCreateTable;
import org.apache.phoenix.parse.DropColumnStatement;
@@ -789,8 +791,8 @@ public class PhoenixStatement implements PhoenixMonitoredStatement, SQLCloseable
private static class ExecutableExplainStatement extends ExplainStatement implements CompilableStatement {
- public ExecutableExplainStatement(BindableStatement statement) {
- super(statement);
+ ExecutableExplainStatement(BindableStatement statement, ExplainType explainType) {
+ super(statement, explainType);
}
@Override
@@ -816,6 +818,13 @@ public class PhoenixStatement implements PhoenixMonitoredStatement, SQLCloseable
}
final StatementPlan plan = compilePlan;
List<String> planSteps = plan.getExplainPlan().getPlanSteps();
+ ExplainType explainType = getExplainType();
+ if (explainType == ExplainType.DEFAULT) {
+ List<String> updatedExplainPlanSteps = new ArrayList<>(planSteps);
+ updatedExplainPlanSteps.removeIf(planStep -> planStep != null
+ && planStep.contains(ExplainTable.REGION_LOCATIONS));
+ planSteps = Collections.unmodifiableList(updatedExplainPlanSteps);
+ }
List<Tuple> tuples = Lists.newArrayListWithExpectedSize(planSteps.size());
Long estimatedBytesToScan = plan.getEstimatedBytesToScan();
Long estimatedRowsToScan = plan.getEstimatedRowsToScan();
@@ -1914,8 +1923,8 @@ public class PhoenixStatement implements PhoenixMonitoredStatement, SQLCloseable
}
@Override
- public ExplainStatement explain(BindableStatement statement) {
- return new ExecutableExplainStatement(statement);
+ public ExplainStatement explain(BindableStatement statement, ExplainType explainType) {
+ return new ExecutableExplainStatement(statement, explainType);
}
@Override
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java
index 49ce5737a4..3b28ca5c0d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainStatement.java
@@ -21,9 +21,11 @@ import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
public class ExplainStatement implements BindableStatement {
private final BindableStatement statement;
-
- public ExplainStatement(BindableStatement statement) {
+ private final ExplainType explainType;
+
+ public ExplainStatement(BindableStatement statement, ExplainType explainType) {
this.statement = statement;
+ this.explainType = explainType;
}
public BindableStatement getStatement() {
@@ -39,4 +41,8 @@ public class ExplainStatement implements BindableStatement {
public Operation getOperation() {
return Operation.QUERY;
}
+
+ public ExplainType getExplainType() {
+ return explainType;
+ }
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java
new file mode 100644
index 0000000000..fc35939d35
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ExplainType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.parse;
+
+/**
+ * Explain type attributes used to differentiate output of the explain plan.
+ */
+public enum ExplainType {
+ WITH_REGIONS,
+ DEFAULT
+}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
index 77350cc067..660598b424 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
@@ -216,8 +216,8 @@ public class ParseNodeFactory {
return "$" + tempAliasCounter.incrementAndGet();
}
- public ExplainStatement explain(BindableStatement statement) {
- return new ExplainStatement(statement);
+ public ExplainStatement explain(BindableStatement statement, ExplainType explainType) {
+ return new ExplainStatement(statement, explainType);
}
public AliasedNode aliasedNode(String alias, ParseNode expression) {
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index da9da65ef1..e4885a7910 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -403,6 +403,14 @@ public interface QueryServices extends SQLCloseable {
* Region server holding the SYSTEM.CATALOG table in batch oriented jobs.
*/
String SKIP_SYSTEM_TABLES_EXISTENCE_CHECK = "phoenix.skip.system.tables.existence.check";
+
+ /**
+ * Config key to represent max region locations to be displayed as part of the Explain plan
+ * output.
+ */
+ String MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN =
+ "phoenix.max.region.locations.size.explain.plan";
+
/**
* Get executor service used for parallel scans
*/
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 697554605f..adb3b55e2f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -57,6 +57,7 @@ import static org.apache.phoenix.query.QueryServices.MASTER_INFO_PORT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MAX_MEMORY_PERC_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MAX_MUTATION_SIZE_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN;
import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_SIZE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MAX_SERVER_CACHE_TIME_TO_LIVE_MS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MAX_SERVER_METADATA_CACHE_SIZE_ATTRIB;
@@ -399,6 +400,7 @@ public class QueryServicesOptions {
public static final int DEFAULT_SCAN_PAGE_SIZE = 32768;
public static final boolean DEFAULT_APPLY_TIME_ZONE_DISPLACMENT = false;
public static final boolean DEFAULT_PHOENIX_TABLE_TTL_ENABLED = true;
+ public static final int DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN = 5;
private final Configuration config;
@@ -489,7 +491,9 @@ public class QueryServicesOptions {
.setIfUnset(INDEX_CREATE_DEFAULT_STATE, DEFAULT_CREATE_INDEX_STATE)
.setIfUnset(SKIP_SYSTEM_TABLES_EXISTENCE_CHECK,
DEFAULT_SKIP_SYSTEM_TABLES_EXISTENCE_CHECK)
- .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE);
+ .setIfUnset(MAX_IN_LIST_SKIP_SCAN_SIZE, DEFAULT_MAX_IN_LIST_SKIP_SCAN_SIZE)
+ .setIfUnset(MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN,
+ DEFAULT_MAX_REGION_LOCATIONS_SIZE_EXPLAIN_PLAN);
// HBase sets this to 1, so we reset it to something more appropriate.
// Hopefully HBase will change this, because we can't know if a user set