You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ji...@apache.org on 2022/12/08 19:00:45 UTC
[phoenix] branch 5.1 updated: PHOENIX-6752 Duplicate expression nodes in extract nodes during WHERE compilation phase leads to poor performance (#1529)
This is an automated email from the ASF dual-hosted git repository.
jisaac pushed a commit to branch 5.1
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/5.1 by this push:
new c5047143ef PHOENIX-6752 Duplicate expression nodes in extract nodes during WHERE compilation phase leads to poor performance (#1529)
c5047143ef is described below
commit c5047143ef96e2c19bdf0b6dd0a5458565b82c70
Author: Jacob Isaac <ja...@gmail.com>
AuthorDate: Thu Dec 8 11:00:40 2022 -0800
PHOENIX-6752 Duplicate expression nodes in extract nodes during WHERE compilation phase leads to poor performance (#1529)
Co-authored-by: Jacob Isaac <ji...@salesforce.com>
---
.../java/org/apache/phoenix/end2end/InListIT.java | 201 +++++++++----
.../java/org/apache/phoenix/end2end/QueryIT.java | 1 -
.../java/org/apache/phoenix/compile/KeyPart.java | 7 +-
.../org/apache/phoenix/compile/WhereOptimizer.java | 74 +++--
.../expression/function/InvertFunction.java | 3 +-
.../expression/function/PrefixFunction.java | 11 +-
.../phoenix/expression/function/RTrimFunction.java | 6 +-
.../expression/function/RoundDateExpression.java | 6 +-
.../function/RoundDecimalExpression.java | 6 +-
.../java/org/apache/phoenix/util/SchemaUtil.java | 4 +-
.../apache/phoenix/compile/WhereOptimizerTest.java | 331 ++++++++++++++++++++-
.../expression/RoundFloorCeilExpressionsTest.java | 6 +-
.../java/org/apache/phoenix/query/BaseTest.java | 2 +-
13 files changed, 533 insertions(+), 125 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java
index 54e5acd30c..c37d16fb0e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/InListIT.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.math.BigInteger;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
@@ -33,20 +34,38 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.sql.Types;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.phoenix.compile.ExpressionCompiler;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
+import org.apache.phoenix.parse.ColumnParseNode;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.schema.types.PDecimal;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.types.PTimestamp;
@@ -59,13 +78,14 @@ import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TypeMismatchException;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.thirdparty.com.google.common.base.Function;
+import org.apache.phoenix.thirdparty.com.google.common.base.Joiner;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.EncodedColumnsUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
-import org.apache.phoenix.thirdparty.com.google.common.base.Function;
-import org.apache.phoenix.thirdparty.com.google.common.base.Joiner;
-import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.After;
@@ -79,6 +99,67 @@ import org.junit.runners.Parameterized;
@Category(ParallelStatsDisabledTest.class)
@RunWith(Parameterized.class)
public class InListIT extends ParallelStatsDisabledIT {
+
+ private static class TestWhereExpressionCompiler extends ExpressionCompiler {
+ private boolean disambiguateWithFamily;
+
+ public TestWhereExpressionCompiler(StatementContext context) {
+ super(context);
+ }
+
+ @Override
+ public Expression visit(ColumnParseNode node) throws SQLException {
+ ColumnRef ref = resolveColumn(node);
+ TableRef tableRef = ref.getTableRef();
+ Expression newColumnExpression = ref.newColumnExpression(node.isTableNameCaseSensitive(), node.isCaseSensitive());
+ if (tableRef.equals(context.getCurrentTable()) && !SchemaUtil.isPKColumn(ref.getColumn())) {
+ byte[] cq = tableRef.getTable().getImmutableStorageScheme() == PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS
+ ? QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES : ref.getColumn().getColumnQualifierBytes();
+ // track the where condition columns. Later we need to ensure the Scan in HRS scans these column CFs
+ context.addWhereConditionColumn(ref.getColumn().getFamilyName().getBytes(), cq);
+ }
+ return newColumnExpression;
+ }
+
+ @Override
+ protected ColumnRef resolveColumn(ColumnParseNode node) throws SQLException {
+ ColumnRef ref = super.resolveColumn(node);
+ if (disambiguateWithFamily) {
+ return ref;
+ }
+ PTable table = ref.getTable();
+ // Track if we need to compare KeyValue during filter evaluation
+ // using column family. If the column qualifier is enough, we
+ // just use that.
+ if (!SchemaUtil.isPKColumn(ref.getColumn())) {
+ if (!EncodedColumnsUtil.usesEncodedColumnNames(table)
+ || ref.getColumn().isDynamic()) {
+ try {
+ table.getColumnForColumnName(ref.getColumn().getName().getString());
+ } catch (AmbiguousColumnException e) {
+ disambiguateWithFamily = true;
+ }
+ } else {
+ for (PColumnFamily columnFamily : table.getColumnFamilies()) {
+ if (columnFamily.getName().equals(ref.getColumn().getFamilyName())) {
+ continue;
+ }
+ try {
+ table.getColumnForColumnQualifier(columnFamily.getName().getBytes(),
+ ref.getColumn().getColumnQualifierBytes());
+ // If we find the same qualifier name with different columnFamily,
+ // then set disambiguateWithFamily to true
+ disambiguateWithFamily = true;
+ break;
+ } catch (ColumnNotFoundException ignore) {
+ }
+ }
+ }
+ }
+ return ref;
+ }
+ }
+
private static boolean isInitialized = false;
private static String tableName = generateUniqueName();
private static String tableName2 = generateUniqueName();
@@ -1904,14 +1985,15 @@ public class InListIT extends ParallelStatsDisabledIT {
};
PDataType[] testTSVarVarPKTypes = new PDataType[] { PTimestamp.INSTANCE, PVarchar.INSTANCE, PVarchar.INSTANCE};
- String baseTableName = String.format("TEST_ENTITY.CUSTOM_T%d", 1);
+ String baseTableName = generateUniqueName();
+ String viewName = String.format("Z_%s", baseTableName);
int tenantId = 1;
int numTestCases = 3;
for (int index=0;index<sortOrders.length;index++) {
// Test Case 1: PK1 = Timestamp, PK2 = Varchar, PK3 = Varchar
- String view1Name = String.format("TEST_ENTITY.Z%d", index*numTestCases+1);
+ String view1Name = String.format("TEST_ENTITY.%s%d", viewName, index*numTestCases+1);
String partition1 = String.format("Z%d", index*numTestCases+1);
createTenantView(tenantId,
baseTableName, view1Name, partition1,
@@ -1919,10 +2001,9 @@ public class InListIT extends ParallelStatsDisabledIT {
testTSVarVarPKTypes[1], sortOrders[index][1],
testTSVarVarPKTypes[2], sortOrders[index][2]);
testTSVarVarPKs(tenantId, view1Name, sortOrders[index]);
- dropTenantViewData(tenantId, view1Name);
// Test Case 2: PK1 = Varchar, PK2 = Varchar, PK3 = Varchar
- String view2Name = String.format("TEST_ENTITY.Z%d", index*numTestCases+2);
+ String view2Name = String.format("TEST_ENTITY.%s%d", viewName, index*numTestCases+2);
String partition2 = String.format("Z%d", index*numTestCases+2);
PDataType[] testVarVarVarPKTypes = new PDataType[] { PVarchar.INSTANCE, PVarchar.INSTANCE, PVarchar.INSTANCE};
createTenantView(tenantId,
@@ -1931,11 +2012,10 @@ public class InListIT extends ParallelStatsDisabledIT {
testVarVarVarPKTypes[1], sortOrders[index][1],
testVarVarVarPKTypes[2], sortOrders[index][2]);
testVarVarVarPKs(tenantId, view2Name, sortOrders[index]);
- dropTenantViewData(tenantId, view2Name);
// Test Case 3: PK1 = Bigint, PK2 = Decimal, PK3 = Bigint
- String view3Name = String.format("TEST_ENTITY.Z%d", index*numTestCases+3);
+ String view3Name = String.format("TEST_ENTITY.%s%d", viewName, index*numTestCases+3);
String partition3 = String.format("Z%d", index*numTestCases+3);
PDataType[] testIntDecIntPKTypes = new PDataType[] { PLong.INSTANCE, PDecimal.INSTANCE, PLong.INSTANCE};
createTenantView(tenantId,
@@ -1944,8 +2024,6 @@ public class InListIT extends ParallelStatsDisabledIT {
testIntDecIntPKTypes[1], sortOrders[index][1],
testIntDecIntPKTypes[2], sortOrders[index][2]);
testIntDecIntPK(tenantId, view3Name, sortOrders[index]);
- dropTenantViewData(tenantId, view3Name);
-
}
}
@@ -2111,8 +2189,6 @@ public class InListIT extends ParallelStatsDisabledIT {
private void assertExpectedWithMaxInList(int tenantId, String testType, PDataType[] testPKTypes, String testSQL, SortOrder[] sortOrder, Set<String> expected) throws SQLException {
String context = "sql: " + testSQL + ", type: " + testType + ", sort-order: " + Arrays.stream(sortOrder).map(s -> s.name()).collect(Collectors.joining(","));
int numInLists = 25;
- int numInListCols = 3;
- Random rnd = new Random();
boolean expectSkipScan = checkMaxSkipScanCardinality ? Arrays.stream(sortOrder).allMatch(Predicate.isEqual(SortOrder.ASC)) : true;
StringBuilder query = new StringBuilder(testSQL);
@@ -2125,54 +2201,8 @@ public class InListIT extends ParallelStatsDisabledIT {
try (Connection tenantConnection = DriverManager.getConnection(tenantConnectionUrl)) {
PhoenixPreparedStatement stmt = tenantConnection.prepareStatement(query.toString()).unwrap(PhoenixPreparedStatement.class);
// perform the query
- for (int i = 0; i<numInLists; i++) {
- for (int b=0;b<testPKTypes.length;b++) {
- int colIndex = i*numInListCols+b+1;
- switch (testPKTypes[b].getSqlType()) {
- case Types.VARCHAR: {
- // pkTypeStr = "VARCHAR(25)";
- int begin = rnd.nextInt(34);
- int remaining = "1234567890abcdefghijklmnopqrstuvwxyz".length() - begin;
- int end = begin + (remaining > 25 ? 25 : remaining);
- stmt.setString(colIndex, "1234567890abcdefghijklmnopqrstuvwxyz".substring(begin, end));
- break;
- }
- case Types.CHAR: {
- //pkTypeStr = "CHAR(15)";
- int begin = rnd.nextInt(34);
- int remaining = "1234567890abcdefghijklmnopqrstuvwxyz".length() - begin;
- int end = begin + (remaining > 15 ? 15 : remaining);
- stmt.setString(colIndex,
- "1234567890abcdefghijklmnopqrstuvwxyz".substring(begin, end));
- break;
- }
- case Types.DECIMAL:
- //pkTypeStr = "DECIMAL(8,2)";
- stmt.setDouble(colIndex, rnd.nextDouble());
- break;
- case Types.INTEGER:
- //pkTypeStr = "INTEGER";
- stmt.setInt(colIndex, rnd.nextInt(50000));
- break;
- case Types.BIGINT:
- //pkTypeStr = "BIGINT";
- stmt.setLong(colIndex, System.currentTimeMillis() + rnd.nextInt(50000));
- break;
- case Types.DATE:
- //pkTypeStr = "DATE";
- stmt.setDate(colIndex, new Date(System.currentTimeMillis() + rnd.nextInt(50000)));
- break;
- case Types.TIMESTAMP:
- //pkTypeStr = "TIMESTAMP";
- stmt.setTimestamp(colIndex, new Timestamp(System.currentTimeMillis() + rnd.nextInt(50000)));
- break;
- default:
- //pkTypeStr = "VARCHAR(25)";
- stmt.setString(colIndex, "1234567890abcdefghijklmnopqrstuvwxyz".substring(rnd.nextInt(34)).substring(0) );
- }
-
- }
- }
+ int lastBoundCol = 0;
+ setBindVariables(stmt, lastBoundCol, numInLists, testPKTypes);
QueryPlan plan = stmt.compileQuery(query.toString());
if (expectSkipScan) {
assertTrue(plan.getExplainPlan().toString().contains("CLIENT PARALLEL 1-WAY POINT LOOKUP ON"));
@@ -2267,4 +2297,53 @@ public class InListIT extends ParallelStatsDisabledIT {
}
}
+ private int setBindVariables(PhoenixPreparedStatement stmt, int startBindIndex, int numBinds, PDataType[] testPKTypes)
+ throws SQLException {
+
+ Random rnd = new Random();
+ int lastBindCol = 0;
+ int numCols = testPKTypes.length;
+ for (int i = 0; i<numBinds; i++) {
+ for (int b=0;b<testPKTypes.length;b++) {
+ int colIndex = startBindIndex + i*numCols+b+1;
+ switch (testPKTypes[b].getSqlType()) {
+ case Types.VARCHAR: {
+ // pkTypeStr = "VARCHAR(25)";
+ stmt.setString(colIndex, RandomStringUtils.randomAlphanumeric(25));
+ break;
+ }
+ case Types.CHAR: {
+ //pkTypeStr = "CHAR(15)";
+ stmt.setString(colIndex, RandomStringUtils.randomAlphanumeric(15));
+ break;
+ }
+ case Types.DECIMAL:
+ //pkTypeStr = "DECIMAL(8,2)";
+ stmt.setDouble(colIndex, rnd.nextDouble());
+ break;
+ case Types.INTEGER:
+ //pkTypeStr = "INTEGER";
+ stmt.setInt(colIndex, rnd.nextInt(50000));
+ break;
+ case Types.BIGINT:
+ //pkTypeStr = "BIGINT";
+ stmt.setLong(colIndex, System.currentTimeMillis() + rnd.nextInt(50000));
+ break;
+ case Types.DATE:
+ //pkTypeStr = "DATE";
+ stmt.setDate(colIndex, new Date(System.currentTimeMillis() + rnd.nextInt(50000)));
+ break;
+ case Types.TIMESTAMP:
+ //pkTypeStr = "TIMESTAMP";
+ stmt.setTimestamp(colIndex, new Timestamp(System.currentTimeMillis() + rnd.nextInt(50000)));
+ break;
+ default:
+ // pkTypeStr = "VARCHAR(25)";
+ stmt.setString(colIndex, RandomStringUtils.randomAlphanumeric(25));
+ }
+ lastBindCol = colIndex;
+ }
+ }
+ return lastBindCol;
+ }
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
index f83a9e0dca..1bee00e2d2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
@@ -168,5 +168,4 @@ public class QueryIT extends BaseQueryIT {
conn.close();
}
}
-
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
index c2d8e7e639..c25b56fe87 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.compile;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.phoenix.expression.Expression;
@@ -61,11 +62,11 @@ public interface KeyPart {
* range (i.e. rows may pass through that will fail when
* the REGEXP_SUBSTR is evaluated).
*
- * @return an empty list if the expression should remain in
- * the WHEERE clause for post filtering or a singleton list
+ * @return an empty set if the expression should remain in
+ * the WHERE clause for post filtering or a singleton set
* containing the expression if it should be removed.
*/
- public List<Expression> getExtractNodes();
+ public Set<Expression> getExtractNodes();
/**
* Gets the primary key column associated with the start of this key part.
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
index 584e96bb42..3f4e6a4ee1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
@@ -25,13 +25,12 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.phoenix.expression.DelegateExpression;
-import org.apache.phoenix.schema.ValueSchema;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
@@ -78,6 +77,7 @@ import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
import org.apache.phoenix.thirdparty.com.google.common.collect.Iterators;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
@@ -240,8 +240,11 @@ public class WhereOptimizer {
boolean isInList = false;
int cnfStartPos = cnf.size();
+ // TODO:
+ // Using keyPart.getExtractNodes() to determine whether the keyPart has a IN List
+ // is not guaranteed, since the IN LIST slot may not have any extracted nodes.
if (keyPart.getExtractNodes() != null && keyPart.getExtractNodes().size() > 0
- && keyPart.getExtractNodes().get(0) instanceof InListExpression){
+ && keyPart.getExtractNodes().iterator().next() instanceof InListExpression){
isInList = true;
}
while (true) {
@@ -389,7 +392,7 @@ public class WhereOptimizer {
// across a multi-range for a prior slot. The reason is that we have an inexact range after
// that, so must filter on the remaining conditions (see issue #467).
if (!stopExtracting) {
- List<Expression> nodesToExtract = keyPart.getExtractNodes();
+ Set<Expression> nodesToExtract = keyPart.getExtractNodes();
extractNodes.addAll(nodesToExtract);
}
}
@@ -645,14 +648,14 @@ public class WhereOptimizer {
if (isDegenerate(keyRanges)) {
return EMPTY_KEY_SLOTS;
}
-
- List<Expression> extractNodes = extractNode == null || slot.getKeyPart().getExtractNodes().isEmpty()
- ? Collections.<Expression>emptyList()
- : Collections.<Expression>singletonList(extractNode);
+
+ Set<Expression> extractNodes = extractNode == null || slot.getKeyPart().getExtractNodes().isEmpty()
+ ? Collections.emptySet()
+ : new LinkedHashSet<>(Collections.<Expression>singleton(extractNode));
return new SingleKeySlot(new BaseKeyPart(table, slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, slot.getOrderPreserving());
}
- private KeySlots newKeyParts(KeySlot slot, List<Expression> extractNodes, List<KeyRange> keyRanges) {
+ private KeySlots newKeyParts(KeySlot slot, Set<Expression> extractNodes, List<KeyRange> keyRanges) {
if (isDegenerate(keyRanges)) {
return EMPTY_KEY_SLOTS;
}
@@ -670,12 +673,12 @@ public class WhereOptimizer {
for (int i = 0; i < childSlots.size(); i++) {
KeySlots slots = childSlots.get(i);
KeySlot keySlot = slots.getSlots().iterator().next();
- List<Expression> childExtractNodes = keySlot.getKeyPart().getExtractNodes();
+ Set<Expression> childExtractNodes = keySlot.getKeyPart().getExtractNodes();
// Stop if there was a gap in extraction of RVC elements. This is required if the leading
// RVC has not row key columns, as we'll still get childSlots if the RVC has trailing row
// key columns. We can't rule the RVC out completely when the childSlots is less the the
// RVC length, as a partial, *leading* match is optimizable.
- if (childExtractNodes.size() != 1 || !childExtractNodes.get(0).equals(rvc.getChildren().get(i))) {
+ if (childExtractNodes.size() != 1 || !childExtractNodes.contains(rvc.getChildren().get(i))) {
break;
}
int pkPosition = keySlot.getPKPosition();
@@ -724,7 +727,7 @@ public class WhereOptimizer {
if (isDegenerate(slot.getKeyRanges())) {
return EMPTY_KEY_SLOTS;
}
- final List<Expression> extractNodes = Collections.<Expression>singletonList(node);
+ final Set<Expression> extractNodes = new LinkedHashSet<>(Collections.<Expression>singletonList(node));
final KeyPart childPart = slot.getKeyPart();
final ImmutableBytesWritable ptr = context.getTempPtr();
return new SingleKeySlot(new CoerceKeySlot(
@@ -856,12 +859,14 @@ public class WhereOptimizer {
* @return
*/
private KeySlots andKeySlots(AndExpression andExpression, List<KeySlots> childSlots) {
+
if(childSlots.isEmpty()) {
return null;
}
// Exit early if it's already been determined that one of the child slots cannot
// possibly be true.
boolean partialExtraction = andExpression.getChildren().size() != childSlots.size();
+
int nChildSlots = childSlots.size();
for (int i = 0; i < nChildSlots; i++) {
KeySlots childSlot = childSlots.get(i);
@@ -894,7 +899,8 @@ public class WhereOptimizer {
for (int pkPos = initPkPos; pkPos < nPkColumns; pkPos++) {
SlotsIterator iterator = new SlotsIterator(childSlots, pkPos);
OrderPreserving orderPreserving = null;
- List<Expression> extractNodes = Lists.newArrayList();
+ Set<KeyPart> visitedKeyParts = Sets.newHashSet();
+ Set<Expression> extractNodes = new LinkedHashSet<>();
List<KeyRange> keyRanges = Lists.newArrayList();
// This is the information carried forward as we process in PK order.
// It's parallel with the list of keyRanges.
@@ -926,7 +932,10 @@ public class WhereOptimizer {
if (slot.getOrderPreserving() != null) {
orderPreserving = slot.getOrderPreserving().combine(orderPreserving);
}
- if (slot.getKeyPart().getExtractNodes() != null) {
+ // Extract once per iteration, when there are large number
+ // of OR clauses (for e.g N > 100k).
+ // The extractNodes.addAll method can get called N times.
+ if (visitedKeyParts.add(slot.getKeyPart()) && slot.getKeyPart().getExtractNodes() != null) {
extractNodes.addAll(slot.getKeyPart().getExtractNodes());
}
// Keep a running intersection of the ranges we see. Note that the
@@ -1007,7 +1016,7 @@ public class WhereOptimizer {
}
}
keySlotArray[pkPos] = new KeySlot(
- new BaseKeyPart(table, table.getPKColumns().get(pkPos), mayExtractNodes ? extractNodes : Collections.<Expression>emptyList()),
+ new BaseKeyPart(table, table.getPKColumns().get(pkPos), mayExtractNodes ? extractNodes : Collections.<Expression>emptySet()),
pkPos,
maxSpan,
keyRanges,
@@ -1063,8 +1072,9 @@ public class WhereOptimizer {
pkSpan = Math.max(pkSpan, rowKeySchema.computeMaxSpan(pkPos, trimmedResult, ptr));
}
}
- List<Expression> extractNodes = mayExtractNodes ?
- keySlotArray[pkPos].getKeyPart().getExtractNodes() : Collections.<Expression>emptyList();
+
+ Set<Expression> extractNodes = mayExtractNodes ?
+ keySlotArray[pkPos].getKeyPart().getExtractNodes() : new LinkedHashSet<>();
keySlotArray[pkPos] = new KeySlot(
new BaseKeyPart(table, table.getPKColumns().get(pkPos), extractNodes),
pkPos,
@@ -1484,7 +1494,7 @@ public class WhereOptimizer {
}
int initialPos = (table.getBucketNum() ==null ? 0 : 1) + (this.context.getConnection().getTenantId() != null && table.isMultiTenant() ? 1 : 0) + (table.getViewIndexId() == null ? 0 : 1);
KeySlot theSlot = null;
- List<Expression> slotExtractNodes = Lists.<Expression>newArrayList();
+ Set<Expression> slotExtractNodes = new LinkedHashSet<>();
int thePosition = -1;
boolean partialExtraction = false;
// TODO: Have separate list for single span versus multi span
@@ -1535,8 +1545,8 @@ public class WhereOptimizer {
theSlot = new KeySlot(new BaseKeyPart(table, table.getPKColumns().get(initialPos), slotExtractNodes), initialPos, 1, EVERYTHING_RANGES, null);
}
return newKeyParts(
- theSlot,
- partialExtraction ? slotExtractNodes : Collections.<Expression>singletonList(orExpression),
+ theSlot,
+ partialExtraction ? slotExtractNodes : new LinkedHashSet<>(Collections.<Expression>singletonList(orExpression)),
slotRanges.isEmpty() ? EVERYTHING_RANGES : KeyRange.coalesce(slotRanges));
}
@@ -1599,7 +1609,7 @@ public class WhereOptimizer {
@Override
public KeySlots visit(RowKeyColumnExpression node) {
PColumn column = table.getPKColumns().get(node.getPosition());
- return new SingleKeySlot(new BaseKeyPart(table, column, Collections.<Expression>singletonList(node)), node.getPosition(), 1, EVERYTHING_RANGES);
+ return new SingleKeySlot(new BaseKeyPart(table, column, new LinkedHashSet<>(Collections.<Expression>singletonList(node))), node.getPosition(), 1, EVERYTHING_RANGES);
}
@Override
@@ -1838,7 +1848,7 @@ public class WhereOptimizer {
return keyRanges;
}
- public final KeySlot concatExtractNodes(List<Expression> extractNodes) {
+ public final KeySlot concatExtractNodes(Set<Expression> extractNodes) {
return new KeySlot(
new BaseKeyPart(this.getKeyPart().getTable(), this.getKeyPart().getColumn(),
SchemaUtil.concat(this.getKeyPart().getExtractNodes(),extractNodes)),
@@ -1941,16 +1951,16 @@ public class WhereOptimizer {
private final PTable table;
private final PColumn column;
- private final List<Expression> nodes;
+ private final Set<Expression> nodes;
- private BaseKeyPart(PTable table, PColumn column, List<Expression> nodes) {
+ private BaseKeyPart(PTable table, PColumn column, Set<Expression> nodes) {
this.table = table;
this.column = column;
this.nodes = nodes;
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
return nodes;
}
@@ -1970,10 +1980,10 @@ public class WhereOptimizer {
private final KeyPart childPart;
private final ImmutableBytesWritable ptr;
private final CoerceExpression node;
- private final List<Expression> extractNodes;
+ private final Set<Expression> extractNodes;
public CoerceKeySlot(KeyPart childPart, ImmutableBytesWritable ptr,
- CoerceExpression node, List<Expression> extractNodes) {
+ CoerceExpression node, Set<Expression> extractNodes) {
this.childPart = childPart;
this.ptr = ptr;
this.node = node;
@@ -2010,7 +2020,7 @@ public class WhereOptimizer {
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
return extractNodes;
}
@@ -2028,24 +2038,24 @@ public class WhereOptimizer {
private class RowValueConstructorKeyPart implements KeyPart {
private final RowValueConstructorExpression rvc;
private final PColumn column;
- private final List<Expression> nodes;
+ private final Set<Expression> nodes;
private final List<KeySlots> childSlots;
private RowValueConstructorKeyPart(PColumn column, RowValueConstructorExpression rvc, int span, List<KeySlots> childSlots) {
this.column = column;
if (span == rvc.getChildren().size()) {
this.rvc = rvc;
- this.nodes = Collections.<Expression>singletonList(rvc);
+ this.nodes = new LinkedHashSet<>(Collections.singletonList(rvc));
this.childSlots = childSlots;
} else {
this.rvc = new RowValueConstructorExpression(rvc.getChildren().subList(0, span),rvc.isStateless());
- this.nodes = Collections.<Expression>emptyList();
+ this.nodes = new LinkedHashSet<>();
this.childSlots = childSlots.subList(0, span);
}
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
return nodes;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
index 038221d738..ef865a9f25 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.expression.function;
import java.sql.SQLException;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -135,7 +136,7 @@ public class InvertFunction extends ScalarFunction {
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
return childPart.getExtractNodes();
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
index b09c14dfc1..34658d0202 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
@@ -20,8 +20,11 @@ package org.apache.phoenix.expression.function;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import java.util.Set;
+
import org.apache.phoenix.compile.KeyPart;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.query.KeyRange;
@@ -54,9 +57,9 @@ abstract public class PrefixFunction extends ScalarFunction {
}
private class PrefixKeyPart implements KeyPart {
- private final List<Expression> extractNodes = extractNode() ?
- Collections.<Expression>singletonList(PrefixFunction.this)
- : Collections.<Expression>emptyList();
+ private final Set<Expression> extractNodes = extractNode() ?
+ new LinkedHashSet<>(Collections.<Expression>singleton(PrefixFunction.this))
+ : Collections.emptySet();
private final KeyPart childPart;
PrefixKeyPart(KeyPart childPart) {
@@ -69,7 +72,7 @@ abstract public class PrefixFunction extends ScalarFunction {
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
return extractNodes;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
index da09eebb65..4d853a07ad 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
@@ -20,7 +20,9 @@ package org.apache.phoenix.expression.function;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -176,10 +178,10 @@ public class RTrimFunction extends ScalarFunction {
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
// We cannot extract the node, as we may have false positives with trailing
// non blank characters such as 'foo bar' where the RHS constant is 'foo'.
- return Collections.<Expression>emptyList();
+ return Collections.emptySet();
}
@Override
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
index 1b77a7e643..dfeba73d3c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
@@ -23,7 +23,9 @@ import java.io.IOException;
import java.sql.Date;
import java.sql.SQLException;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -251,7 +253,7 @@ public class RoundDateExpression extends ScalarFunction {
@Override
public KeyPart newKeyPart(final KeyPart childPart) {
return new KeyPart() {
- private final List<Expression> extractNodes = Collections.<Expression>singletonList(RoundDateExpression.this);
+ private final Set<Expression> extractNodes = new LinkedHashSet<>(Collections.<Expression>singleton(RoundDateExpression.this));
@Override
public PColumn getColumn() {
@@ -259,7 +261,7 @@ public class RoundDateExpression extends ScalarFunction {
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
return extractNodes;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
index c0c34b69db..5f6a1714db 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
@@ -24,7 +24,9 @@ import java.math.BigDecimal;
import java.math.RoundingMode;
import java.sql.SQLException;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -180,7 +182,7 @@ public class RoundDecimalExpression extends ScalarFunction {
@Override
public KeyPart newKeyPart(final KeyPart childPart) {
return new KeyPart() {
- private final List<Expression> extractNodes = Collections.<Expression>singletonList(RoundDecimalExpression.this);
+ private final Set<Expression> extractNodes = new LinkedHashSet<>(Collections.<Expression>singleton(RoundDecimalExpression.this));
@Override
public PColumn getColumn() {
@@ -188,7 +190,7 @@ public class RoundDecimalExpression extends ScalarFunction {
}
@Override
- public List<Expression> getExtractNodes() {
+ public Set<Expression> getExtractNodes() {
return extractNodes;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index bf9a51329c..d9748cd4d1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -264,7 +264,7 @@ public class SchemaUtil {
return name != null && name.length() > 0 && name.charAt(0) == '"' && name.indexOf("\"", 1) == name.length() - 1;
}
- public static <T> List<T> concat(List<T> l1, List<T> l2) {
+ public static <T> Set<T> concat(Set<T> l1, Set<T> l2) {
int size1 = l1.size();
if (size1 == 0) {
return l2;
@@ -273,7 +273,7 @@ public class SchemaUtil {
if (size2 == 0) {
return l1;
}
- List<T> l3 = new ArrayList<T>(size1 + size2);
+ Set<T> l3 = new LinkedHashSet<T>(size1 + size2);
l3.addAll(l1);
l3.addAll(l2);
return l3;
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
index a34cd64be1..4b0dbb3274 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.compile;
import static org.apache.phoenix.query.KeyRange.EVERYTHING_RANGE;
import static org.apache.phoenix.query.KeyRange.getKeyRange;
import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY;
+import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
import static org.apache.phoenix.util.TestUtil.BINARY_NAME;
import static org.apache.phoenix.util.TestUtil.BTABLE_NAME;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
@@ -44,11 +45,42 @@ import java.sql.Date;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Timestamp;
+import java.sql.Types;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.phoenix.expression.AndExpression;
+import org.apache.phoenix.parse.ColumnParseNode;
+import org.apache.phoenix.parse.ParseNode;
+import org.apache.phoenix.parse.SQLParser;
+import org.apache.phoenix.parse.SelectStatement;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.ColumnRef;
+import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PDate;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PDecimal;
+import org.apache.phoenix.schema.types.PDouble;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PTimestamp;
+import org.apache.phoenix.schema.types.PUnsignedLong;
+import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Scan;
@@ -73,19 +105,10 @@ import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.schema.ColumnNotFoundException;
-import org.apache.phoenix.schema.ColumnRef;
-import org.apache.phoenix.schema.SortOrder;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PDate;
-import org.apache.phoenix.schema.types.PDecimal;
-import org.apache.phoenix.schema.types.PDouble;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.schema.types.PLong;
-import org.apache.phoenix.schema.types.PUnsignedLong;
-import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ScanUtil;
@@ -95,7 +118,69 @@ import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
-
+
+ private static class TestWhereExpressionCompiler extends ExpressionCompiler {
+ private boolean disambiguateWithFamily;
+
+ public TestWhereExpressionCompiler(StatementContext context) {
+ super(context);
+ }
+
+ @Override
+ public Expression visit(ColumnParseNode node) throws SQLException {
+ ColumnRef ref = resolveColumn(node);
+ TableRef tableRef = ref.getTableRef();
+ Expression newColumnExpression = ref.newColumnExpression(node.isTableNameCaseSensitive(), node.isCaseSensitive());
+ if (tableRef.equals(context.getCurrentTable()) && !SchemaUtil.isPKColumn(ref.getColumn())) {
+ byte[] cq = tableRef.getTable().getImmutableStorageScheme() == PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS
+ ? QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES : ref.getColumn().getColumnQualifierBytes();
+ // track the where condition columns. Later we need to ensure the Scan in HRS scans these column CFs
+ context.addWhereConditionColumn(ref.getColumn().getFamilyName().getBytes(), cq);
+ }
+ return newColumnExpression;
+ }
+
+ @Override
+ protected ColumnRef resolveColumn(ColumnParseNode node) throws SQLException {
+ ColumnRef ref = super.resolveColumn(node);
+ if (disambiguateWithFamily) {
+ return ref;
+ }
+ PTable table = ref.getTable();
+ // Track if we need to compare KeyValue during filter evaluation
+ // using column family. If the column qualifier is enough, we
+ // just use that.
+ if (!SchemaUtil.isPKColumn(ref.getColumn())) {
+ if (!EncodedColumnsUtil.usesEncodedColumnNames(table)
+ || ref.getColumn().isDynamic()) {
+ try {
+ table.getColumnForColumnName(ref.getColumn().getName().getString());
+ } catch (AmbiguousColumnException e) {
+ disambiguateWithFamily = true;
+ }
+ } else {
+ for (PColumnFamily columnFamily : table.getColumnFamilies()) {
+ if (columnFamily.getName().equals(ref.getColumn().getFamilyName())) {
+ continue;
+ }
+ try {
+ table.getColumnForColumnQualifier(columnFamily.getName().getBytes(),
+ ref.getColumn().getColumnQualifierBytes());
+ // If we find the same qualifier name with different columnFamily,
+ // then set disambiguateWithFamily to true
+ disambiguateWithFamily = true;
+ break;
+ } catch (ColumnNotFoundException ignore) {
+ }
+ }
+ }
+ }
+ return ref;
+ }
+ }
+
+ private static final String TENANT_PREFIX = "Txt00tst1";
+
private static StatementContext compileStatement(String query) throws SQLException {
return compileStatement(query, Collections.emptyList(), null);
}
@@ -3236,4 +3321,226 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
"(PK5, TO_INTEGER(PK6), PK7) < (5, TO_INTEGER(TO_INTEGER(6)), 7))"));
}
}
+
+
+ @Test
+ public void testWithLargeORs() throws Exception {
+
+ SortOrder[][] sortOrders = new SortOrder[][] {
+ {SortOrder.ASC, SortOrder.ASC, SortOrder.ASC},
+ {SortOrder.ASC, SortOrder.ASC, SortOrder.DESC},
+ {SortOrder.ASC, SortOrder.DESC, SortOrder.ASC},
+ {SortOrder.ASC, SortOrder.DESC, SortOrder.DESC},
+ {SortOrder.DESC, SortOrder.ASC, SortOrder.ASC},
+ {SortOrder.DESC, SortOrder.ASC, SortOrder.DESC},
+ {SortOrder.DESC, SortOrder.DESC, SortOrder.ASC},
+ {SortOrder.DESC, SortOrder.DESC, SortOrder.DESC}
+ };
+
+ String tableName = generateUniqueName();
+ String viewName = String.format("Z_%s", tableName);
+ PDataType[] testTSVarVarPKTypes = new PDataType[] { PTimestamp.INSTANCE, PVarchar.INSTANCE, PInteger.INSTANCE};
+ String baseTableName = String.format("TEST_ENTITY.%s", tableName);
+ int tenantId = 1;
+ int numTestCases = 1;
+ for (int index=0;index<sortOrders.length;index++) {
+ // Test Case 1: PK1 = Timestamp, PK2 = Varchar, PK3 = Integer
+ String view1Name = String.format("TEST_ENTITY.%s%d",
+ viewName, index*numTestCases+1);
+ String partition1 = String.format("Z%d", index*numTestCases+1);
+ createTenantView(tenantId,
+ baseTableName, view1Name, partition1,
+ testTSVarVarPKTypes[0], sortOrders[index][0],
+ testTSVarVarPKTypes[1], sortOrders[index][1],
+ testTSVarVarPKTypes[2], sortOrders[index][2]);
+ testTSVarIntAndLargeORs(tenantId, view1Name, sortOrders[index]);
+ }
+ }
+
+ private void createBaseTable(String baseTable) throws SQLException {
+
+ try (Connection globalConnection = DriverManager.getConnection(getUrl())) {
+ try (Statement cstmt = globalConnection.createStatement()) {
+ String CO_BASE_TBL_TEMPLATE = "CREATE TABLE IF NOT EXISTS %s(OID CHAR(15) NOT NULL,KP CHAR(3) NOT NULL,ROW_ID VARCHAR, COL1 VARCHAR,COL2 VARCHAR,COL3 VARCHAR,CREATED_DATE DATE,CREATED_BY CHAR(15),LAST_UPDATE DATE,LAST_UPDATE_BY CHAR(15),SYSTEM_MODSTAMP DATE CONSTRAINT pk PRIMARY KEY (OID,KP)) MULTI_TENANT=true,COLUMN_ENCODED_BYTES=0";
+ cstmt.execute(String.format(CO_BASE_TBL_TEMPLATE, baseTable));
+ }
+ }
+ return;
+ }
+
+ private void createTenantView(int tenant, String baseTable, String tenantView, String partition,
+ PDataType pkType1, SortOrder pk1Order,
+ PDataType pkType2, SortOrder pk2Order,
+ PDataType pkType3, SortOrder pk3Order) throws SQLException {
+
+ String pkType1Str = getType(pkType1);
+ String pkType2Str = getType(pkType2);
+ String pkType3Str = getType(pkType3);
+ createBaseTable(baseTable);
+
+ String tenantConnectionUrl = String.format("%s;%s=%s%06d", getUrl(), TENANT_ID_ATTRIB, TENANT_PREFIX, tenant);
+ try (Connection tenantConnection = DriverManager.getConnection(tenantConnectionUrl)) {
+ try (Statement cstmt = tenantConnection.createStatement()) {
+ String TENANT_VIEW_TEMPLATE = "CREATE VIEW IF NOT EXISTS %s(ID1 %s not null,ID2 %s not null,ID3 %s not null,COL4 VARCHAR,COL5 VARCHAR,COL6 VARCHAR CONSTRAINT pk PRIMARY KEY (ID1 %s, ID2 %s, ID3 %s)) "
+ + "AS SELECT * FROM %s WHERE KP = '%s'";
+ cstmt.execute(String.format(TENANT_VIEW_TEMPLATE, tenantView, pkType1Str, pkType2Str, pkType3Str, pk1Order.name(),pk2Order.name(),pk3Order.name(), baseTable, partition));
+ }
+ }
+ return;
+ }
+
+ private int setBindVariables(PhoenixPreparedStatement stmt, int startBindIndex, int numBinds, PDataType[] testPKTypes)
+ throws SQLException {
+
+ Random rnd = new Random();
+ int lastBindCol = 0;
+ int numCols = testPKTypes.length;
+ for (int i = 0; i<numBinds; i++) {
+ for (int b=0;b<testPKTypes.length;b++) {
+ int colIndex = startBindIndex + i*numCols+b+1;
+ switch (testPKTypes[b].getSqlType()) {
+ case Types.VARCHAR: {
+ // pkTypeStr = "VARCHAR(25)";
+ stmt.setString(colIndex, RandomStringUtils.randomAlphanumeric(25));
+ break;
+ }
+ case Types.CHAR: {
+ //pkTypeStr = "CHAR(15)";
+ stmt.setString(colIndex, RandomStringUtils.randomAlphanumeric(15));
+ break;
+ }
+ case Types.DECIMAL:
+ //pkTypeStr = "DECIMAL(8,2)";
+ stmt.setDouble(colIndex, rnd.nextDouble());
+ break;
+ case Types.INTEGER:
+ //pkTypeStr = "INTEGER";
+ stmt.setInt(colIndex, rnd.nextInt(50000));
+ break;
+ case Types.BIGINT:
+ //pkTypeStr = "BIGINT";
+ stmt.setLong(colIndex, System.currentTimeMillis() + rnd.nextInt(50000));
+ break;
+ case Types.DATE:
+ //pkTypeStr = "DATE";
+ stmt.setDate(colIndex, new Date(System.currentTimeMillis() + rnd.nextInt(50000)));
+ break;
+ case Types.TIMESTAMP:
+ //pkTypeStr = "TIMESTAMP";
+ stmt.setTimestamp(colIndex, new Timestamp(System.currentTimeMillis() + rnd.nextInt(50000)));
+ break;
+ default:
+ // pkTypeStr = "VARCHAR(25)";
+ stmt.setString(colIndex, RandomStringUtils.randomAlphanumeric(25));
+ }
+ lastBindCol = colIndex;
+ }
+ }
+ return lastBindCol;
+ }
+
+ private String getType(PDataType pkType) {
+ String pkTypeStr = "VARCHAR(25)";
+ switch (pkType.getSqlType()) {
+ case Types.VARCHAR:
+ pkTypeStr = "VARCHAR(25)";
+ break;
+ case Types.CHAR:
+ pkTypeStr = "CHAR(15)";
+ break;
+ case Types.DECIMAL:
+ pkTypeStr = "DECIMAL(8,2)";
+ break;
+ case Types.INTEGER:
+ pkTypeStr = "INTEGER";
+ break;
+ case Types.BIGINT:
+ pkTypeStr = "BIGINT";
+ break;
+ case Types.DATE:
+ pkTypeStr = "DATE";
+ break;
+ case Types.TIMESTAMP:
+ pkTypeStr = "TIMESTAMP";
+ break;
+ default:
+ pkTypeStr = "VARCHAR(25)";
+ }
+ return pkTypeStr;
+ }
+ // Test Case 1: PK1 = Timestamp, PK2 = Varchar, PK3 = Integer
+ private void testTSVarIntAndLargeORs(int tenantId, String viewName, SortOrder[] sortOrder) throws SQLException {
+ String testName = "testLargeORs";
+ String testLargeORs = String.format("SELECT ROW_ID FROM %s ", viewName);
+
+ PDataType[] testPKTypes = new PDataType[] { PTimestamp.INSTANCE, PVarchar.INSTANCE, PInteger.INSTANCE};
+ assertExpectedWithMaxInListAndLargeORs(tenantId, testName, testPKTypes, testLargeORs, sortOrder);
+
+ }
+
+ public void assertExpectedWithMaxInListAndLargeORs(int tenantId, String testType, PDataType[] testPKTypes, String testSQL, SortOrder[] sortOrder) throws SQLException {
+
+ Properties tenantProps = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ int numINs = 25;
+ int expectedExtractedNodes = Arrays.asList(new SortOrder[] {sortOrder[0], sortOrder[1]}).stream().allMatch(Predicate.isEqual(SortOrder.ASC)) ? 3 : 2;
+
+ // Test for increasing orders of ORs (5,50,500,5000)
+ for (int o = 0; o < 4;o++) {
+ int numORs = (int) (5.0 * Math.pow(10.0,(double) o));
+ String context = "ORs:" + numORs + ", sql: " + testSQL + ", type: " + testType + ", sort-order: " + Arrays.stream(sortOrder).map(s -> s.name()).collect(Collectors.joining(","));
+ String tenantConnectionUrl = String.format("%s;%s=%s%06d", getUrl(), TENANT_ID_ATTRIB, TENANT_PREFIX, tenantId);
+ try (Connection tenantConnection = DriverManager.getConnection(tenantConnectionUrl, tenantProps)) {
+ // Generate the where clause
+ StringBuilder whereClause = new StringBuilder("(ID1,ID2) IN ((?,?)");
+ for (int i = 0; i < numINs; i++) {
+ whereClause.append(",(?,?)");
+ }
+ whereClause.append(") AND (ID3 = ? ");
+ for (int i = 0; i < numORs; i++) {
+ whereClause.append(" OR ID3 = ?");
+ }
+ whereClause.append(") LIMIT 200");
+ // Full SQL
+ String query = testSQL + " WHERE " + whereClause;
+
+ PhoenixPreparedStatement stmtForExtractNodesCheck =
+ tenantConnection.prepareStatement(query).unwrap(PhoenixPreparedStatement.class);
+ int lastBoundCol = 0;
+ lastBoundCol = setBindVariables(stmtForExtractNodesCheck, lastBoundCol,
+ numINs + 1,
+ new PDataType[] {testPKTypes[0], testPKTypes[1]});
+ lastBoundCol = setBindVariables(stmtForExtractNodesCheck, lastBoundCol,
+ numORs + 1, new PDataType[] {testPKTypes[2]});
+
+ // Get the column resolver
+ SelectStatement selectStatement = new SQLParser(query).parseQuery();
+ ColumnResolver resolver = FromCompiler.getResolverForQuery(selectStatement,
+ tenantConnection.unwrap(PhoenixConnection.class));
+
+ // Where clause with INs and ORs
+ ParseNode whereNode = selectStatement.getWhere();
+ Expression whereExpression = whereNode.accept(new TestWhereExpressionCompiler(
+ new StatementContext(stmtForExtractNodesCheck, resolver)));
+
+ // Tenant view where clause
+ ParseNode viewWhere = SQLParser.parseCondition("KP = 'ECZ'");
+ Expression viewWhereExpression = viewWhere.accept(new TestWhereExpressionCompiler(
+ new StatementContext(stmtForExtractNodesCheck, resolver)));
+
+ // Build the test expression
+ Expression testExpression = AndExpression.create(
+ Lists.newArrayList(whereExpression, viewWhereExpression));
+
+ // Test
+ Set<Expression> extractedNodes = Sets.<Expression>newHashSet();
+ WhereOptimizer.pushKeyExpressionsToScan(new StatementContext(stmtForExtractNodesCheck, resolver),
+ Collections.emptySet(), testExpression, extractedNodes, Optional.<byte[]>absent());
+ assertEquals(String.format("Unexpected results expected = %d, actual = %d extracted nodes",
+ expectedExtractedNodes, extractedNodes.size()),
+ expectedExtractedNodes, extractedNodes.size());
+ }
+
+ }
+ }
+
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java
index 1a349efeed..e0420c3795 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java
@@ -31,7 +31,9 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -267,8 +269,8 @@ public class RoundFloorCeilExpressionsTest extends BaseConnectionlessQueryTest {
}
@Override
- public List<Expression> getExtractNodes() {
- return Collections.emptyList();
+ public Set<Expression> getExtractNodes() {
+ return Collections.emptySet();
}
@Override
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 3e28ba6145..7436092ec9 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -1681,7 +1681,7 @@ public abstract class BaseTest {
LOGGER.info("Disabled and dropped {} tables in {} ms", tableCount, endTime-startTime);
}
-
+
public static void assertOneOfValuesEqualsResultSet(ResultSet rs, List<List<Object>>... expectedResultsArray) throws SQLException {
List<List<Object>> results = Lists.newArrayList();
while (rs.next()) {