You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2022/02/25 18:17:35 UTC
[phoenix] branch 4.x updated: PHOENIX-6458 Using global indexes for queries with uncovered columns (#1256)
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new c1aa9f1 PHOENIX-6458 Using global indexes for queries with uncovered columns (#1256)
c1aa9f1 is described below
commit c1aa9f1b18ea8be94aa96bce04fd57e2ec715aaf
Author: kadirozde <37...@users.noreply.github.com>
AuthorDate: Thu Feb 24 14:56:24 2022 -0800
PHOENIX-6458 Using global indexes for queries with uncovered columns (#1256)
---
.../end2end/index/GlobalIndexCheckerIT.java | 122 +++++++++++++++++++++
.../end2end/index/GlobalIndexOptimizationIT.java | 78 ++++++-------
.../apache/phoenix/compile/ExpressionCompiler.java | 11 +-
.../org/apache/phoenix/compile/FromCompiler.java | 14 ++-
.../org/apache/phoenix/compile/JoinCompiler.java | 11 +-
.../apache/phoenix/compile/ProjectionCompiler.java | 17 ++-
.../apache/phoenix/compile/StatementContext.java | 9 ++
.../phoenix/compile/TupleProjectionCompiler.java | 23 ++--
.../org/apache/phoenix/compile/WhereCompiler.java | 9 +-
.../coprocessor/BaseScannerRegionObserver.java | 7 ++
.../GroupedAggregateRegionObserver.java | 12 +-
.../org/apache/phoenix/execute/BaseQueryPlan.java | 27 +++--
.../phoenix/iterate/BaseResultIterators.java | 7 +-
.../org/apache/phoenix/iterate/ExplainTable.java | 6 +-
.../iterate/NonAggregateRegionScannerFactory.java | 24 ++--
.../phoenix/iterate/RegionScannerFactory.java | 23 ++--
.../apache/phoenix/optimize/QueryOptimizer.java | 1 +
...xDataColumnRef.java => IndexDataColumnRef.java} | 41 +++++--
.../java/org/apache/phoenix/schema/TableRef.java | 32 ++++--
.../java/org/apache/phoenix/util/IndexUtil.java | 62 +++++++++--
.../java/org/apache/phoenix/util/ScanUtil.java | 11 ++
21 files changed, 403 insertions(+), 144 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
index cb01149..02c57e2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
@@ -306,6 +306,44 @@ public class GlobalIndexCheckerIT extends BaseTest {
assertEquals("ae", rs.getString(1));
assertEquals("efg", rs.getString(2));
assertFalse(rs.next());
+ conn.createStatement().execute("DROP INDEX " + indexTableName + " on " +
+ dataTableName);
+ // Run the previous test on an uncovered global index
+ indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " +
+ dataTableName + " (PHOENIX_ROW_TIMESTAMP())" +
+ (async ? "ASYNC" : "")+ this.indexDDLOptions);
+ if (async) {
+ // Run the index MR job to rebuild the index and verify that index is built correctly
+ IndexToolIT.runIndexTool(false, null, dataTableName,
+ indexTableName, null, 0, IndexTool.IndexVerifyType.AFTER);
+ }
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName, query);
+
+ // Verify that we will read from the index table with the index hint
+ query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ " +
+ "val1, val2, PHOENIX_ROW_TIMESTAMP() from " + dataTableName + " WHERE " +
+ "PHOENIX_ROW_TIMESTAMP() > TO_DATE('" + initial.toString() + "','yyyy-MM-dd HH:mm:ss.SSS', '" + timeZoneID + "')";
+
+ assertExplainPlan(conn, query, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("ab", rs.getString(1));
+ assertEquals("abc", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("bc", rs.getString(1));
+ assertEquals("bcd", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("bc", rs.getString(1));
+ assertEquals("ccc", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("de", rs.getString(1));
+ assertEquals("def", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ae", rs.getString(1));
+ assertEquals("efg", rs.getString(2));
+ assertFalse(rs.next());
}
}
@@ -430,6 +468,90 @@ public class GlobalIndexCheckerIT extends BaseTest {
}
}
+ private void assertIndexTableNotSelected(Connection conn, String dataTableName, String indexTableName, String sql)
+ throws Exception {
+ try {
+ assertExplainPlan(conn, sql, dataTableName, indexTableName);
+ throw new AssertionError("The index table should not be selected without an index hint");
+ } catch (AssertionError error){
+ //expected
+ }
+ }
+
+ @Test
+ public void testUncoveredGlobalIndex() throws Exception {
+ if (async) {
+ return;
+ }
+ String dataTableName = generateUniqueName();
+ populateTable(dataTableName); // with two rows ('a', 'ab', 'abc', 'abcd') and ('b', 'bc', 'bcd', 'bcde')
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " +
+ dataTableName + " (val1) include (val2)" + this.indexDDLOptions);
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT val3 from " + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')");
+
+ //Verify that with index hint, we will read from the index table even though val3 is not included by the index table
+ String selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val3 from "
+ + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')";
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("bcde", rs.getString(1));
+ assertFalse(rs.next());
+ conn.createStatement().execute("DROP INDEX " + indexTableName + " on " + dataTableName);
+ // Create an index does not include any columns
+ indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexTableName + " on " +
+ dataTableName + " (val1)" + this.indexDDLOptions);
+ conn.commit();
+
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT id from " + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')");
+ selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ id from " + dataTableName + " WHERE val1 = 'bc' AND (val2 = 'bcd' OR val3 ='bcde')";
+ //Verify that we will read from the index table
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertFalse(rs.next());
+
+ // Add another row and run a group by query where the uncovered index should be used
+ conn.createStatement().execute("upsert into " + dataTableName + " (id, val1, val2, val3) values ('c', 'ab','cde', 'cdef')");
+ conn.commit();
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT count(*) from " + dataTableName + " where val1 > '0' GROUP BY val1");
+ selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ count(*) from " + dataTableName + " where val1 > '0' GROUP BY val1";
+ //Verify that we will read from the index table
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertFalse(rs.next());
+ // Run an order by query where the uncovered index should be used
+ // Verify that without hint, the index table is not selected
+ assertIndexTableNotSelected(conn, dataTableName, indexTableName,
+ "SELECT val3 from " + dataTableName + " where val1 > '0' ORDER BY val1");
+ selectSql = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ val3 from " + dataTableName + " where val1 > '0' ORDER BY val1";
+ //Verify that we will read from the index table
+ assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
+ rs = conn.createStatement().executeQuery(selectSql);
+ assertTrue(rs.next());
+ assertEquals("abcd", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("cdef", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("bcde", rs.getString(1));
+ assertFalse(rs.next());
+ }
+ }
+
@Test
public void testSimulateConcurrentUpdates() throws Exception {
try (Connection conn = DriverManager.getConnection(getUrl())) {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java
index 96e83f5..99f5997 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java
@@ -230,17 +230,12 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where v1<='z' and k3 > 1 order by V1,t_id";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
- expected =
- "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName + "\n" +
- " SERVER FILTER BY K3 > 1\n" +
- " SERVER SORTED BY \\[" + dataTableName + ".V1, " + dataTableName + ".T_ID\\]\n" +
- "CLIENT MERGE SORT\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\[\\*\\] - \\['z'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
+ expected =
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n"+
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY AND \"K3\" > 1";
actual = QueryUtil.getExplainPlan(rs);
- assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual));
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -265,17 +260,15 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, V1, k3 from " + dataTableFullName + " where v1 <='z' group by v1,t_id, k3";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
-
- expected =
- "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName + "\n" +
- " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[" + dataTableName + ".V1, " + dataTableName + ".T_ID, " + dataTableName + ".K3\\]\n" +
- "CLIENT MERGE SORT\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\[\\*\\] - \\['z'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
+ expected =
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n"+
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]\n"+
+ "CLIENT MERGE SORT";
+
actual = QueryUtil.getExplainPlan(rs);
- assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual));
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -299,16 +292,13 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ v1,sum(k3) from " + dataTableFullName + " where v1 <='z' group by v1 order by v1";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
- expected =
- "CLIENT PARALLEL \\d-WAY FULL SCAN OVER " + dataTableName + "\n" +
- " SERVER AGGREGATE INTO DISTINCT ROWS BY \\[" + dataTableName + ".V1\\]\n" +
- "CLIENT MERGE SORT\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\[\\*\\] - \\['z'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".T_ID\", \"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
+ expected =
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " [*] - ['z']\n"+
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]";
actual = QueryUtil.getExplainPlan(rs);
- assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual));
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -340,12 +330,10 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
ResultSet rs = conn1.createStatement().executeQuery("EXPLAIN "+ query);
String actual = QueryUtil.getExplainPlan(rs);
- String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + dataTableName + " \\['tid1'\\]\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " \\['tid1','a'\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" + dataTableName + ".K1\", \"" + dataTableName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
- assertTrue("Expected:\n" + expected + "\ndid not match\n" + actual, Pattern.matches(expected, actual));
+ String expected = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + indexTableName + " ['tid1','a']\n" +
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY";
+ assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, actual.equals(expected));
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -393,15 +381,15 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
* This inner "_IDX_" + dataTableName use skipScan, and all the whereExpressions are already in SkipScanFilter,
* so there is no other RowKeyComparisonFilter needed.
*/
+
String actual = QueryUtil.getExplainPlan(rs);
String expected =
- "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + dataTableName + "\n" +
- " SERVER FILTER BY V1 = 'a'\n" +
- " SKIP-SCAN-JOIN TABLE 0\n" +
- " CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER _IDX_" + dataTableName + " \\[" + Short.MIN_VALUE + ",1\\] - \\[" + Short.MIN_VALUE + ",2\\]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " DYNAMIC SERVER FILTER BY \\(\"" + viewName + ".T_ID\", \"" + viewName + ".K1\", \"" + viewName + ".K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)";
- assertTrue("Expected:\n" + expected + "\ndid not match\n" + actual, Pattern.matches(expected,actual));
+ "CLIENT PARALLEL 1-WAY SKIP SCAN ON 2 KEYS OVER _IDX_" + dataTableName +
+ " [" + Short.MIN_VALUE + ",1] - [" + Short.MIN_VALUE + ",2]\n" +
+ " SERVER MERGE [0.K3]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY";
+
+ assertEquals(expected,actual);
rs = conn1.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -409,7 +397,7 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
assertEquals(2, rs.getInt("k1"));
assertEquals(4, rs.getInt("k2"));
assertEquals(2, rs.getInt("k3"));
- assertEquals("a", rs.getString("v1"));
+ assertEquals("a", rs.getString(5)); //TODO use name v1 instead of position 5, see PHOENIX-6644
assertFalse(rs.next());
} finally {
conn1.close();
@@ -474,7 +462,7 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
assertFalse(rs.next());
// No where clause
- query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " order by V1,t_id";
+ query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName + " order by V1,t_id";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
assertEquals(
@@ -511,7 +499,7 @@ public class GlobalIndexOptimizationIT extends ParallelStatsDisabledIT {
assertFalse(rs.next());
// No where clause in index scan
- query = "SELECT /*+ INDEX(" + dataTableName + " " + indexTableName + ")*/ t_id, k1, k2, k3, V1 from " + dataTableFullName + " where k3 > 1 order by V1,t_id";
+ query = "SELECT t_id, k1, k2, k3, V1 from " + dataTableFullName + " where k3 > 1 order by V1,t_id";
rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
assertEquals(
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
index d1b44df..986cdf3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ExpressionCompiler.java
@@ -112,7 +112,7 @@ import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
import org.apache.phoenix.schema.DelegateDatum;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PDatum;
import org.apache.phoenix.schema.PTable;
@@ -140,6 +140,8 @@ import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
+
public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor<Expression> {
private boolean isAggregate;
@@ -371,9 +373,12 @@ public class ExpressionCompiler extends UnsupportedAllParseNodeVisitor<Expressio
// Rather than not use a local index when a column not contained by it is referenced, we
// join back to the data table in our coprocessor since this is a relatively cheap
// operation given that we know the join is local.
- if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL) {
+ if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(context.getCurrentTable())) {
try {
- return new LocalIndexDataColumnRef(context, context.getCurrentTable(), node.getName());
+ context.setUncoveredIndex(true);
+ return new IndexDataColumnRef(context, context.getCurrentTable(),
+ node.getName());
} catch (ColumnFamilyNotFoundException c) {
throw e;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index 197a0a6..88d80aa 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -103,6 +103,7 @@ import org.apache.phoenix.thirdparty.com.google.common.collect.ListMultimap;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import static org.apache.phoenix.monitoring.MetricType.NUM_METADATA_LOOKUP_FAILURES;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
/**
* Validates FROM clause and builds a ColumnResolver for resolving column references
@@ -1157,13 +1158,14 @@ public class FromCompiler {
}
private static class ProjectedTableColumnResolver extends MultiTableColumnResolver {
- private final boolean isLocalIndex;
+ private final boolean isIndex;
private final List<TableRef> theTableRefs;
private final Map<ColumnRef, Integer> columnRefMap;
private ProjectedTableColumnResolver(PTable projectedTable, PhoenixConnection conn, Map<String, UDFParseNode> udfParseNodes) throws SQLException {
super(conn, 0, udfParseNodes, null);
Preconditions.checkArgument(projectedTable.getType() == PTableType.PROJECTED);
- this.isLocalIndex = projectedTable.getIndexType() == IndexType.LOCAL;
+ this.isIndex = projectedTable.getIndexType() == IndexType.LOCAL
+ || projectedTable.getIndexType() == IndexType.GLOBAL;
this.columnRefMap = new HashMap<ColumnRef, Integer>();
long ts = Long.MAX_VALUE;
for (int i = projectedTable.getBucketNum() == null ? 0 : 1; i < projectedTable.getColumns().size(); i++) {
@@ -1201,9 +1203,11 @@ public class FromCompiler {
try {
colRef = super.resolveColumn(schemaName, tableName, colName);
} catch (ColumnNotFoundException e) {
- // This could be a ColumnRef for local index data column.
- TableRef tableRef = isLocalIndex ? super.getTables().get(0) : super.resolveTable(schemaName, tableName);
- if (tableRef.getTable().getIndexType() == IndexType.LOCAL) {
+ // This could be a ColumnRef for index data column.
+ TableRef tableRef = isIndex ? super.getTables().get(0)
+ : super.resolveTable(schemaName, tableName);
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
try {
TableRef parentTableRef = super.resolveTable(
tableRef.getTable().getSchemaName().getString(),
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
index de49826..70345b5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.compile;
import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -75,7 +76,7 @@ import org.apache.phoenix.parse.TableNodeVisitor;
import org.apache.phoenix.parse.TableWildcardParseNode;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.MetaDataEntityNotFoundException;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PNameFactory;
@@ -1127,7 +1128,8 @@ public class JoinCompiler {
if (columnRef.getTableRef().equals(tableRef)
&& (!retainPKColumns || !SchemaUtil.isPKColumn(columnRef.getColumn()))) {
if (columnRef instanceof LocalIndexColumnRef) {
- sourceColumns.add(new LocalIndexDataColumnRef(context, tableRef, IndexUtil.getIndexColumnName(columnRef.getColumn())));
+ sourceColumns.add(new IndexDataColumnRef(context, tableRef,
+ IndexUtil.getIndexColumnName(columnRef.getColumn())));
} else {
sourceColumns.add(columnRef);
}
@@ -1392,10 +1394,11 @@ public class JoinCompiler {
try {
columnRef = resolver.resolveColumn(node.getSchemaName(), node.getTableName(), node.getName());
} catch (ColumnNotFoundException e) {
- // This could be a LocalIndexDataColumnRef. If so, the table name must have
+ // This could be an IndexDataColumnRef. If so, the table name must have
// been appended by the IndexStatementRewriter, and we can convert it into.
TableRef tableRef = resolver.resolveTable(node.getSchemaName(), node.getTableName());
- if (tableRef.getTable().getIndexType() == IndexType.LOCAL) {
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
TableRef parentTableRef = FromCompiler.getResolver(
NODE_FACTORY.namedTable(null, TableName.create(tableRef.getTable()
.getSchemaName().getString(), tableRef.getTable()
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
index c13f383..f36f605 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.compile;
import static org.apache.phoenix.query.QueryServices.WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
@@ -69,9 +70,9 @@ import org.apache.phoenix.schema.ArgumentTypeMismatchException;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.KeyValueSchema;
import org.apache.phoenix.schema.KeyValueSchema.KeyValueSchemaBuilder;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PDatum;
@@ -230,9 +231,11 @@ public class ProjectionCompiler {
indexColumn = index.getColumnForColumnName(indexColName);
ref = new ColumnRef(tableRef, indexColumn.getPosition());
} catch (ColumnNotFoundException e) {
- if (index.getIndexType() == IndexType.LOCAL) {
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
try {
- ref = new LocalIndexDataColumnRef(context, tableRef, indexColName);
+ context.setUncoveredIndex(true);
+ ref = new IndexDataColumnRef(context, tableRef, indexColName);
indexColumn = ref.getColumn();
} catch (ColumnFamilyNotFoundException c) {
throw e;
@@ -303,9 +306,11 @@ public class ProjectionCompiler {
ref = new ColumnRef(tableRef, indexColumn.getPosition());
indexColumnFamily = indexColumn.getFamilyName() == null ? null : indexColumn.getFamilyName().getString();
} catch (ColumnNotFoundException e) {
- if (index.getIndexType() == IndexType.LOCAL) {
+ if (tableRef.getTable().getIndexType() == IndexType.LOCAL
+ || isHintedGlobalIndex(tableRef)) {
try {
- ref = new LocalIndexDataColumnRef(context, tableRef, indexColName);
+ context.setUncoveredIndex(true);
+ ref = new IndexDataColumnRef(context, tableRef, indexColName);
indexColumn = ref.getColumn();
indexColumnFamily =
indexColumn.getFamilyName() == null ? null
@@ -702,7 +707,7 @@ public class ProjectionCompiler {
PColumn col = expression.getColumn();
// hack'ish... For covered columns with local indexes we defer to the server.
if (col instanceof ProjectedColumn && ((ProjectedColumn) col)
- .getSourceColumnRef() instanceof LocalIndexDataColumnRef) {
+ .getSourceColumnRef() instanceof IndexDataColumnRef) {
return null;
}
PTable table = context.getCurrentTable().getTable();
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
index 2abd546..d6cc20b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -87,6 +87,7 @@ public class StatementContext {
private final OverAllQueryMetrics overAllQueryMetrics;
private QueryLogger queryLogger;
private boolean isClientSideUpsertSelect;
+ private boolean isUncoveredIndex;
public StatementContext(PhoenixStatement statement) {
this(statement, new Scan());
@@ -330,6 +331,14 @@ public class StatementContext {
this.isClientSideUpsertSelect = isClientSideUpsertSelect;
}
+ public boolean isUncoveredIndex() {
+ return isUncoveredIndex;
+ }
+
+ public void setUncoveredIndex(boolean isUncoveredIndex) {
+ this.isUncoveredIndex = isUncoveredIndex;
+ }
+
/*
* setRetryingPersistentCache can be used to override the USE_PERSISTENT_CACHE hint and disable the use of the
* persistent cache for a specific cache ID. This can be used to retry queries that failed when using the persistent
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 2a67d8d..9174f50 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.compile;
import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+import static org.apache.phoenix.util.IndexUtil.isHintedGlobalIndex;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -41,13 +42,12 @@ import org.apache.phoenix.parse.WildcardParseNode;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
-import org.apache.phoenix.schema.LocalIndexDataColumnRef;
+import org.apache.phoenix.schema.IndexDataColumnRef;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.EncodedCQCounter;
-import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.ProjectedColumn;
@@ -154,14 +154,18 @@ public class TupleProjectionCompiler {
EncodedColumnsUtil.setColumns(column, table, context.getScan());
}
}
- // add LocalIndexDataColumnRef
+ // add IndexDataColumnRef
position = projectedColumns.size();
- for (LocalIndexDataColumnRef sourceColumnRef : visitor.localIndexColumnRefSet) {
+ for (IndexDataColumnRef sourceColumnRef : visitor.indexColumnRefSet) {
PColumn column = new ProjectedColumn(sourceColumnRef.getColumn().getName(),
sourceColumnRef.getColumn().getFamilyName(), position++,
sourceColumnRef.getColumn().isNullable(), sourceColumnRef, sourceColumnRef.getColumn().getColumnQualifierBytes());
projectedColumns.add(column);
}
+ if (!visitor.indexColumnRefSet.isEmpty()
+ && tableRef.isHinted()) {
+ context.setUncoveredIndex(true);
+ }
return PTableImpl.builderWithColumns(table, projectedColumns)
.setType(PTableType.PROJECTED)
.setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
@@ -230,12 +234,12 @@ public class TupleProjectionCompiler {
private static class ColumnRefVisitor extends StatelessTraverseAllParseNodeVisitor {
private final StatementContext context;
private final LinkedHashSet<ColumnRef> nonPkColumnRefSet;
- private final LinkedHashSet<LocalIndexDataColumnRef> localIndexColumnRefSet;
+ private final LinkedHashSet<IndexDataColumnRef> indexColumnRefSet;
private ColumnRefVisitor(StatementContext context) {
this.context = context;
this.nonPkColumnRefSet = new LinkedHashSet<ColumnRef>();
- this.localIndexColumnRefSet = new LinkedHashSet<LocalIndexDataColumnRef>();
+ this.indexColumnRefSet = new LinkedHashSet<IndexDataColumnRef>();
}
@Override
@@ -247,9 +251,12 @@ public class TupleProjectionCompiler {
nonPkColumnRefSet.add(resolveColumn);
}
} catch (ColumnNotFoundException e) {
- if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL) {
+ if (context.getCurrentTable().getTable().getIndexType() == PTable.IndexType.LOCAL
+ || isHintedGlobalIndex(context.getCurrentTable())) {
try {
- localIndexColumnRefSet.add(new LocalIndexDataColumnRef(context, context.getCurrentTable(), node.getName()));
+ context.setUncoveredIndex(true);
+ indexColumnRefSet.add(new IndexDataColumnRef(context,
+ context.getCurrentTable(), node.getName()));
} catch (ColumnFamilyNotFoundException c) {
throw e;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java
index 9439a3c..1f6ab7f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereCompiler.java
@@ -278,7 +278,9 @@ public class WhereCompiler {
if (LiteralExpression.isBooleanFalseOrNull(whereClause)) {
context.setScanRanges(ScanRanges.NOTHING);
- } else if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL) {
+ } else if (context.getCurrentTable().getTable().getIndexType() == IndexType.LOCAL
+ || (context.getCurrentTable().getTable().getIndexType() == IndexType.GLOBAL
+ && context.isUncoveredIndex())) {
if (whereClause != null && !ExpressionUtil.evaluatesToTrue(whereClause)) {
// pass any extra where as scan attribute so it can be evaluated after all
// columns from the main CF have been merged in
@@ -291,11 +293,12 @@ public class WhereCompiler {
} catch (IOException e) {
throw new RuntimeException(e);
}
- scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER, stream.toByteArray());
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_FILTER, stream.toByteArray());
// this is needed just for ExplainTable, since de-serializing an expression does not restore
// its display properties, and that cannot be changed, due to backwards compatibility
- scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR, Bytes.toBytes(whereClause.toString()));
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_FILTER_STR,
+ Bytes.toBytes(whereClause.toString()));
}
} else if (whereClause != null && !ExpressionUtil.evaluatesToTrue(whereClause)) {
Filter filter = null;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index 9a63e88..64e2dae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -86,6 +86,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
public static final String GROUP_BY_LIMIT = "_GroupByLimit";
public static final String LOCAL_INDEX = "_LocalIndex";
public static final String LOCAL_INDEX_BUILD = "_LocalIndexBuild";
+ public static final String UNCOVERED_GLOBAL_INDEX = "_UncoveredGlobalIndex";
public static final String INDEX_REBUILD_PAGING = "_IndexRebuildPaging";
// The number of index rows to be rebuild in one RPC call
public static final String INDEX_REBUILD_PAGE_ROWS = "_IndexRebuildPageRows";
@@ -97,9 +98,15 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver {
"_IndexRebuildDisableLoggingVerifyType";
public static final String INDEX_REBUILD_DISABLE_LOGGING_BEYOND_MAXLOOKBACK_AGE =
"_IndexRebuildDisableLoggingBeyondMaxLookbackAge";
+ @Deprecated
public static final String LOCAL_INDEX_FILTER = "_LocalIndexFilter";
+ @Deprecated
public static final String LOCAL_INDEX_LIMIT = "_LocalIndexLimit";
+ @Deprecated
public static final String LOCAL_INDEX_FILTER_STR = "_LocalIndexFilterStr";
+ public static final String INDEX_FILTER = "_IndexFilter";
+ public static final String INDEX_LIMIT = "_IndexLimit";
+ public static final String INDEX_FILTER_STR = "_IndexFilterStr";
/*
* Attribute to denote that the index maintainer has been serialized using its proto-buf presentation.
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
index 00fd405..306f1fe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
@@ -136,13 +136,8 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
.getEnvironment().getConfiguration(), em);
RegionScanner innerScanner = s;
- boolean useProto = false;
- byte[] localIndexBytes = scan.getAttribute(LOCAL_INDEX_BUILD_PROTO);
- useProto = localIndexBytes != null;
- if (localIndexBytes == null) {
- localIndexBytes = scan.getAttribute(LOCAL_INDEX_BUILD);
- }
- List<IndexMaintainer> indexMaintainers = localIndexBytes == null ? null : IndexMaintainer.deserialize(localIndexBytes, useProto);
+ List<IndexMaintainer> indexMaintainers =
+ IndexUtil.deSerializeIndexMaintainersFromScan(scan);
TupleProjector tupleProjector = null;
byte[][] viewConstants = null;
ColumnReference[] dataColumns = IndexUtil.deserializeDataTableColumnsToJoin(scan);
@@ -150,7 +145,8 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan);
final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan));
- if (ScanUtil.isLocalIndex(scan) || (j == null && p != null)) {
+ if (ScanUtil.isLocalOrUncoveredGlobalIndex(scan)
+ || (j == null && p != null)) {
if (dataColumns != null) {
tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index d9aacd7..5494ca8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.execute;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.sql.ParameterMetaData;
import java.sql.SQLException;
import java.util.Collections;
@@ -325,11 +324,16 @@ public abstract class BaseQueryPlan implements QueryPlan {
ScanUtil.setTenantId(scan, tenantIdBytes);
String customAnnotations = LogUtil.customAnnotationsToString(connection);
- ScanUtil.setCustomAnnotations(scan, customAnnotations == null ? null
- : customAnnotations.getBytes(StandardCharsets.UTF_8));
- // Set local index related scan attributes.
- if (table.getIndexType() == IndexType.LOCAL) {
- ScanUtil.setLocalIndex(scan);
+ ScanUtil.setCustomAnnotations(scan,
+ customAnnotations == null ? null : customAnnotations.getBytes());
+ // Set index related scan attributes.
+ if (table.getType() == PTableType.INDEX) {
+ if (table.getIndexType() == IndexType.LOCAL) {
+ ScanUtil.setLocalIndex(scan);
+ } else if (context.isUncoveredIndex()) {
+ ScanUtil.setUncoveredGlobalIndex(scan);
+ }
+
Set<PColumn> dataColumns = context.getDataColumns();
// If any data columns to join back from data table are present then we set following attributes
// 1. data columns to be projected and their key value schema.
@@ -353,11 +357,12 @@ public abstract class BaseQueryPlan implements QueryPlan {
KeyValueSchema schema = ProjectedColumnExpression.buildSchema(dataColumns);
// Set key value schema of the data columns.
serializeSchemaIntoScan(scan, schema);
-
- // Set index maintainer of the local index.
- serializeIndexMaintainerIntoScan(scan, dataTable);
- // Set view constants if exists.
- serializeViewConstantsIntoScan(scan, dataTable);
+ if (table.getIndexType() == IndexType.LOCAL) {
+ // Set index maintainer of the local index.
+ serializeIndexMaintainerIntoScan(scan, dataTable);
+ // Set view constants if exists.
+ serializeViewConstantsIntoScan(scan, dataTable);
+ }
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 59df237..26fee20 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -281,13 +281,14 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
}
if (perScanLimit != null) {
- if (scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER) == null) {
+ if (scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER) == null) {
ScanUtil.andFilterAtEnd(scan, new PageFilter(perScanLimit));
} else {
- // if we have a local index filter and a limit, handle the limit after the filter
+ // if we have an index filter and a limit, handle the limit after the filter
// we cast the limit to a long even though it passed as an Integer so that
// if we need extend this in the future the serialization is unchanged
- scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT, Bytes.toBytes((long)perScanLimit));
+ scan.setAttribute(BaseScannerRegionObserver.INDEX_LIMIT,
+ Bytes.toBytes((long) perScanLimit));
}
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
index c3ab8f1..e0256c9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
@@ -191,7 +191,11 @@ public abstract class ExplainTable {
if (whereFilter != null) {
whereFilterStr = whereFilter.toString();
} else {
- byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR);
+ byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER_STR);
+ if (expBytes == null) {
+ // For older clients
+ expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER_STR);
+ }
if (expBytes != null) {
whereFilterStr = Bytes.toString(expBytes);
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
index e258e2a..f371206 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/NonAggregateRegionScannerFactory.java
@@ -128,21 +128,15 @@ public class NonAggregateRegionScannerFactory extends RegionScannerFactory {
PhoenixTransactionContext tx = null;
ColumnReference[] dataColumns = IndexUtil.deserializeDataTableColumnsToJoin(scan);
if (dataColumns != null) {
- tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
- dataRegion = env.getRegion();
- boolean useProto = false;
- byte[] localIndexBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO);
- useProto = localIndexBytes != null;
- if (localIndexBytes == null) {
- localIndexBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD);
- }
- int clientVersion = ScanUtil.getClientVersion(scan);
- List<IndexMaintainer> indexMaintainers =
- IndexMaintainer.deserialize(localIndexBytes, useProto);
- indexMaintainer = indexMaintainers.get(0);
- viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
- byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE);
- tx = TransactionFactory.getTransactionContext(txState, clientVersion);
+ tupleProjector = IndexUtil.getTupleProjector(scan, dataColumns);
+ dataRegion = env.getRegion();
+ int clientVersion = ScanUtil.getClientVersion(scan);
+ List<IndexMaintainer> indexMaintainers =
+ IndexUtil.deSerializeIndexMaintainersFromScan(scan);
+ indexMaintainer = indexMaintainers.get(0);
+ viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan);
+ byte[] txState = scan.getAttribute(BaseScannerRegionObserver.TX_STATE);
+ tx = TransactionFactory.getTransactionContext(txState, clientVersion);
}
final TupleProjector p = TupleProjector.deserializeProjectorFromScan(scan);
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
index 3426de8..8e2bde6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/RegionScannerFactory.java
@@ -135,9 +135,13 @@ public abstract class RegionScannerFactory {
long extraLimit = -1;
{
- // for local indexes construct the row filter for uncovered columns if it exists
- if (ScanUtil.isLocalIndex(scan)) {
- byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER);
+ // for indexes construct the row filter for uncovered columns if it exists
+ if (ScanUtil.isLocalOrUncoveredGlobalIndex(scan)) {
+ byte[] expBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_FILTER);
+ if (expBytes == null) {
+ // For older clients
+ expBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_FILTER);
+ }
if (expBytes != null) {
try {
ByteArrayInputStream stream = new ByteArrayInputStream(expBytes);
@@ -149,7 +153,11 @@ public abstract class RegionScannerFactory {
throw new RuntimeException(io);
}
}
- byte[] limitBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT);
+ byte[] limitBytes = scan.getAttribute(BaseScannerRegionObserver.INDEX_LIMIT);
+ if (limitBytes == null) {
+ // For older clients
+ limitBytes = scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX_LIMIT);
+ }
if (limitBytes != null) {
extraLimit = Bytes.toLong(limitBytes);
}
@@ -217,7 +225,8 @@ public abstract class RegionScannerFactory {
if (result.size() == 0) {
return next;
}
- if (ScanUtil.isLocalIndex(scan) && !ScanUtil.isAnalyzeTable(scan)) {
+ if ((ScanUtil.isLocalOrUncoveredGlobalIndex(scan))
+ && !ScanUtil.isAnalyzeTable(scan)) {
if(actualStartKey!=null) {
next = scanTillScanStartRow(s, arrayKVRefs, arrayFuncRefs, result,
null);
@@ -229,8 +238,8 @@ public abstract class RegionScannerFactory {
dataRegion will never be null in case of non-coprocessor call,
therefore no need to refactor
*/
- IndexUtil.wrapResultUsingOffset(env, result, offset, dataColumns,
- tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr);
+ IndexUtil.wrapResultUsingOffset(env, result, scan, offset, dataColumns,
+ tupleProjector, dataRegion, indexMaintainer, viewConstants, ptr);
if (extraWhere != null) {
Tuple merged = useQualifierAsListIndex ? new PositionBasedResultTuple(result) :
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index ad21d9c..bd3282c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -334,6 +334,7 @@ public class QueryOptimizer {
boolean isProjected = dataPlan.getContext().getResolver().getTables().get(0).getTable().getType() == PTableType.PROJECTED;
// Check index state of now potentially updated index table to make sure it's active
TableRef indexTableRef = resolver.getTables().get(0);
+ indexTableRef.setHinted(isHinted);
PTable indexTable = indexTableRef.getTable();
PIndexState indexState = indexTable.getIndexState();
Map<TableRef, QueryPlan> dataPlans = Collections.singletonMap(indexTableRef, dataPlan);
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java
similarity index 65%
rename from phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java
rename to phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java
index 87f0999..3f45ebb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/LocalIndexDataColumnRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/IndexDataColumnRef.java
@@ -23,17 +23,23 @@ import java.util.Set;
import org.apache.phoenix.compile.FromCompiler;
import org.apache.phoenix.compile.StatementContext;
import org.apache.phoenix.expression.ColumnExpression;
+import org.apache.phoenix.expression.IsNullExpression;
import org.apache.phoenix.expression.ProjectedColumnExpression;
import org.apache.phoenix.parse.ParseNodeFactory;
import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.util.IndexUtil;
-public class LocalIndexDataColumnRef extends ColumnRef {
+/**
+ * Even when a column is not covered by an index table for a given query, we may still want to
+ * use index in the query plan and fetch the missing columns from the data table rows on the
+ * server side. This class is used to keep track of such data columns.
+ */
+public class IndexDataColumnRef extends ColumnRef {
final private int position;
final private Set<PColumn> columns;
private static final ParseNodeFactory FACTORY = new ParseNodeFactory();
- public LocalIndexDataColumnRef(StatementContext context, TableRef tRef, String indexColumnName)
+ public IndexDataColumnRef(StatementContext context, TableRef tRef, String indexColumnName)
throws MetaDataEntityNotFoundException, SQLException {
super(FromCompiler.getResolver(
FACTORY.namedTable(
@@ -48,15 +54,15 @@ public class LocalIndexDataColumnRef extends ColumnRef {
columns = context.getDataColumns();
}
- protected LocalIndexDataColumnRef(LocalIndexDataColumnRef localIndexDataColumnRef, long timestamp) {
- super(localIndexDataColumnRef, timestamp);
- this.position = localIndexDataColumnRef.position;
- this.columns = localIndexDataColumnRef.columns;
+ protected IndexDataColumnRef(IndexDataColumnRef indexDataColumnRef, long timestamp) {
+ super(indexDataColumnRef, timestamp);
+ this.position = indexDataColumnRef.position;
+ this.columns = indexDataColumnRef.columns;
}
@Override
public ColumnRef cloneAtTimestamp(long timestamp) {
- return new LocalIndexDataColumnRef(this, timestamp);
+ return new IndexDataColumnRef(this, timestamp);
}
@Override
@@ -64,4 +70,25 @@ public class LocalIndexDataColumnRef extends ColumnRef {
String displayName = this.getTableRef().getColumnDisplayName(this, schemaNameCaseSensitive, colNameCaseSensitive);
return new ProjectedColumnExpression(this.getColumn(), columns, position, displayName);
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + position;
+ result = prime * result + ((columns == null) ? 0 : columns.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o)) {
+ return false;
+ }
+ IndexDataColumnRef that = (IndexDataColumnRef) o;
+ if (position != that.position) {
+ return false;
+ }
+ return columns.equals(that.columns);
+ }
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
index 5f426b0..1df722f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -38,6 +38,7 @@ public class TableRef {
private final long lowerBoundTimeStamp;
private final boolean hasDynamicCols;
private final long currentTime;
+ private boolean hinted;
private static TableRef createEmptyTableRef() {
try {
@@ -53,15 +54,18 @@ public class TableRef {
}
public TableRef(TableRef tableRef) {
- this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
+ this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp,
+ tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols, tableRef.hinted);
}
public TableRef(TableRef tableRef, long timeStamp) {
- this(tableRef.alias, tableRef.table, timeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
+ this(tableRef.alias, tableRef.table, timeStamp, tableRef.lowerBoundTimeStamp,
+ tableRef.hasDynamicCols, tableRef.hinted);
}
public TableRef(TableRef tableRef, String alias) {
- this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
+ this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp,
+ tableRef.hasDynamicCols, tableRef.hinted);
}
public TableRef(PTable table) {
@@ -69,15 +73,20 @@ public class TableRef {
}
public TableRef(PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp) {
- this(null, table, upperBoundTimeStamp, lowerBoundTimeStamp, false);
+ this(null, table, upperBoundTimeStamp, lowerBoundTimeStamp, false, false);
}
public TableRef(String alias, PTable table, long upperBoundTimeStamp, boolean hasDynamicCols) {
- this(alias, table, upperBoundTimeStamp, 0, hasDynamicCols);
+ this(alias, table, upperBoundTimeStamp, 0, hasDynamicCols, false);
+ }
+
+ public TableRef(String alias, PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp,
+ boolean hasDynamicCols) {
+ this(alias, table, upperBoundTimeStamp, lowerBoundTimeStamp, hasDynamicCols, false);
}
- public TableRef(String alias, PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp,
- boolean hasDynamicCols) {
+ public TableRef(String alias, PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp,
+ boolean hasDynamicCols, boolean hinted) {
this.alias = alias;
this.table = table;
// if UPDATE_CACHE_FREQUENCY is set, always let the server set timestamps
@@ -85,6 +94,7 @@ public class TableRef {
this.currentTime = this.upperBoundTimeStamp;
this.lowerBoundTimeStamp = lowerBoundTimeStamp;
this.hasDynamicCols = hasDynamicCols;
+ this.hinted = hinted;
}
public PTable getTable() {
@@ -103,6 +113,14 @@ public class TableRef {
return alias;
}
+ public boolean isHinted() {
+ return hinted;
+ }
+
+ public void setHinted(boolean hinted) {
+ this.hinted = hinted;
+ }
+
public String getColumnDisplayName(ColumnRef ref, boolean cfCaseSensitive, boolean cqCaseSensitive) {
String cf = null;
String cq = null;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 0bab683..c172f68 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -17,6 +17,9 @@
*/
package org.apache.phoenix.util;
+import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.LOCAL_INDEX_BUILD;
+import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO;
+import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.PHYSICAL_DATA_TABLE_NAME;
import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MAJOR_VERSION;
import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MINOR_VERSION;
import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_PATCH_NUMBER;
@@ -46,6 +49,8 @@ import org.apache.hadoop.hbase.PhoenixTagType;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.phoenix.hbase.index.table.HTableFactory;
+import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.thirdparty.com.google.common.cache.Cache;
import org.apache.phoenix.thirdparty.com.google.common.cache.CacheBuilder;
@@ -459,6 +464,25 @@ public class IndexUtil {
}
}
+ public static List<IndexMaintainer> deSerializeIndexMaintainersFromScan(Scan scan) {
+ boolean useProto = false;
+ byte[] indexBytes = scan.getAttribute(LOCAL_INDEX_BUILD_PROTO);
+ useProto = indexBytes != null;
+ if (indexBytes == null) {
+ indexBytes = scan.getAttribute(LOCAL_INDEX_BUILD);
+ }
+ if (indexBytes == null) {
+ indexBytes = scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD);
+ useProto = indexBytes != null;
+ }
+ if (indexBytes == null) {
+ indexBytes = scan.getAttribute(PhoenixIndexCodec.INDEX_MD);
+ }
+ List<IndexMaintainer> indexMaintainers =
+ indexBytes == null ? null : IndexMaintainer.deserialize(indexBytes, useProto);
+ return indexMaintainers;
+ }
+
public static byte[][] deserializeViewConstantsFromScan(Scan scan) {
byte[] bytes = scan.getAttribute(BaseScannerRegionObserver.VIEW_CONSTANTS);
if (bytes == null) return null;
@@ -556,7 +580,7 @@ public class IndexUtil {
}
public static void wrapResultUsingOffset(final RegionCoprocessorEnvironment environment,
- List<Cell> result, final int offset, ColumnReference[] dataColumns,
+ List<Cell> result, final Scan scan, final int offset, ColumnReference[] dataColumns,
TupleProjector tupleProjector, Region dataRegion, IndexMaintainer indexMaintainer,
byte[][] viewConstants, ImmutableBytesWritable ptr) throws IOException {
if (tupleProjector != null) {
@@ -576,18 +600,27 @@ public class IndexUtil {
}
}
Result joinResult = null;
- if (dataRegion != null) {
- joinResult = dataRegion.get(get);
- } else {
- TableName dataTable =
- TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(
- environment.getRegion().getTableDesc().getNameAsString()));
- HTableInterface table = null;
- try {
- table = environment.getTable(dataTable);
+ if (ScanUtil.isLocalIndex(scan)) {
+ if (dataRegion != null) {
+ joinResult = dataRegion.get(get);
+ } else {
+ TableName dataTable =
+ TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(
+ environment.getRegion().getTableDesc().getNameAsString()));
+ try (Table table = environment.getTable(dataTable)) {
+ joinResult = table.get(get);
+ }
+ }
+ } else if (ScanUtil.isUncoveredGlobalIndex(scan)) {
+ byte[] dataTableName = scan.getAttribute(PHYSICAL_DATA_TABLE_NAME);
+
+ HTableFactory hTableFactory = ServerUtil.getDelegateHTableFactory(environment,
+ ServerUtil.ConnectionType.INDEX_WRITER_CONNECTION);
+ try (Table table = hTableFactory.
+ getTable(new ImmutableBytesPtr(dataTableName))) {
joinResult = table.get(get);
} finally {
- if (table != null) table.close();
+ hTableFactory.shutdown();
}
}
// at this point join result has data from the data table. We now need to take this result and
@@ -981,6 +1014,13 @@ public class IndexUtil {
}
}
+ public static boolean isHintedGlobalIndex(final TableRef tableRef) {
+ PTable table = tableRef.getTable();
+ return table.getType() == PTableType.INDEX
+ && table.getIndexType() == PTable.IndexType.GLOBAL
+ && tableRef.isHinted();
+ }
+
/**
* Updates the EMPTY cell value to VERIFIED for global index table rows.
*/
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index c1d3468..8901c50 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -138,9 +138,20 @@ public class ScanUtil {
scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX, PDataType.TRUE_BYTES);
}
+ public static void setUncoveredGlobalIndex(Scan scan) {
+ scan.setAttribute(BaseScannerRegionObserver.UNCOVERED_GLOBAL_INDEX, PDataType.TRUE_BYTES);
+ }
+
public static boolean isLocalIndex(Scan scan) {
return scan.getAttribute(BaseScannerRegionObserver.LOCAL_INDEX) != null;
}
+ public static boolean isUncoveredGlobalIndex(Scan scan) {
+ return scan.getAttribute(BaseScannerRegionObserver.UNCOVERED_GLOBAL_INDEX) != null;
+ }
+
+ public static boolean isLocalOrUncoveredGlobalIndex(Scan scan) {
+ return isLocalIndex(scan) || isUncoveredGlobalIndex(scan);
+ }
public static boolean isNonAggregateScan(Scan scan) {
return scan.getAttribute(BaseScannerRegionObserver.NON_AGGREGATE_QUERY) != null;