You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2018/07/11 14:29:17 UTC
[1/4] phoenix git commit: PHOENIX-3383 Comparison between descending
row keys used in RVC is reverse
Repository: phoenix
Updated Branches:
refs/heads/master 6195f8e7b -> aee568beb
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
index a7b5687..877c939 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java
@@ -177,7 +177,7 @@ public class PhoenixTxIndexMutationGenerator {
// Project empty key value column
scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), emptyKeyValueQualifier);
- ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
+ ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, null, true, -1);
scanRanges.initializeScan(scan);
Table txTable = indexMetaData.getTransactionContext().getTransactionalTable(htable, isImmutable);
// For rollback, we need to see all versions, including
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FunctionExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FunctionExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FunctionExpression.java
index b45706a..bc9fa9f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FunctionExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FunctionExpression.java
@@ -30,7 +30,15 @@ import org.apache.phoenix.expression.Expression;
* @since 0.1
*/
public abstract class FunctionExpression extends BaseCompoundExpression {
- public enum OrderPreserving {NO, YES_IF_LAST, YES};
+ public enum OrderPreserving {NO, YES_IF_LAST, YES;
+
+ public OrderPreserving combine(OrderPreserving that) {
+ if (that == null) {
+ return this;
+ }
+ return OrderPreserving.values()[Math.min(this.ordinal(), that.ordinal())];
+ }};
+
public FunctionExpression() {
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
index 3615cbe..8ef5914 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/InvertFunction.java
@@ -96,7 +96,24 @@ public class InvertFunction extends ScalarFunction {
@Override
public KeyRange getKeyRange(CompareOp op, Expression rhs) {
KeyRange range = childPart.getKeyRange(op, rhs);
- return range.invert();
+ byte[] lower = range.getLowerRange();
+ if (!range.lowerUnbound()) {
+ lower = SortOrder.invert(lower, 0, lower.length);
+ }
+ byte[] upper;
+ if (range.isSingleKey()) {
+ upper = lower;
+ } else {
+ upper = range.getUpperRange();
+ if (!range.upperUnbound()) {
+ upper = SortOrder.invert(upper, 0, upper.length);
+ }
+ }
+ range = KeyRange.getKeyRange(lower, range.isLowerInclusive(), upper, range.isUpperInclusive());
+ if (getColumn().getSortOrder() == SortOrder.DESC) {
+ range = range.invert();
+ }
+ return range;
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
index cb98e28..ff3e74d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/PrefixFunction.java
@@ -110,7 +110,11 @@ abstract public class PrefixFunction extends ScalarFunction {
lowerRange[lowerRange.length-1] = QueryConstants.SEPARATOR_BYTE;
}
}
- return KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, false);
+ KeyRange range = KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, false);
+ if (column.getSortOrder() == SortOrder.DESC) {
+ range = range.invert();
+ }
+ return range;
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
index 5713713..81e4f9e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RTrimFunction.java
@@ -168,7 +168,11 @@ public class RTrimFunction extends ScalarFunction {
upperRange = type.pad(upperRange, length, SortOrder.ASC);
}
}
- return KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ KeyRange range = KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ if (getColumn().getSortOrder() == SortOrder.DESC) {
+ range = range.invert();
+ }
+ return range;
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
index d747771..988b027 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDateExpression.java
@@ -32,6 +32,9 @@ import org.apache.phoenix.compile.KeyPart;
import org.apache.phoenix.expression.Determinism;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.LiteralExpression;
+import org.apache.phoenix.parse.FunctionParseNode.Argument;
+import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
+import org.apache.phoenix.parse.FunctionParseNode.FunctionClassType;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PTable;
@@ -39,13 +42,10 @@ import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDataType.PDataCodec;
+import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.util.ByteUtil;
-import org.apache.phoenix.schema.types.PDate;
-import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction;
-import org.apache.phoenix.parse.FunctionParseNode.Argument;
-import org.apache.phoenix.parse.FunctionParseNode.FunctionClassType;
import com.google.common.collect.Lists;
@@ -261,6 +261,7 @@ public class RoundDateExpression extends ScalarFunction {
int offset = ByteUtil.isInclusive(op) ? 1 : 0;
long value = codec.decodeLong(key, 0, SortOrder.getDefault());
byte[] nextKey = new byte[type.getByteSize()];
+ KeyRange range;
switch (op) {
case EQUAL:
// If the value isn't evenly divisible by the div amount, then it
@@ -272,18 +273,25 @@ public class RoundDateExpression extends ScalarFunction {
return KeyRange.EMPTY_RANGE;
}
codec.encodeLong(value + divBy, nextKey, 0);
- return type.getKeyRange(key, true, nextKey, false);
+ range = type.getKeyRange(key, true, nextKey, false);
+ break;
case GREATER:
case GREATER_OR_EQUAL:
codec.encodeLong((value + divBy - offset)/divBy*divBy, nextKey, 0);
- return type.getKeyRange(nextKey, true, KeyRange.UNBOUND, false);
+ range = type.getKeyRange(nextKey, true, KeyRange.UNBOUND, false);
+ break;
case LESS:
case LESS_OR_EQUAL:
codec.encodeLong((value + divBy - (1 -offset))/divBy*divBy, nextKey, 0);
- return type.getKeyRange(KeyRange.UNBOUND, false, nextKey, false);
+ range = type.getKeyRange(KeyRange.UNBOUND, false, nextKey, false);
+ break;
default:
return childPart.getKeyRange(op, rhs);
}
+ if (getColumn().getSortOrder() == SortOrder.DESC) {
+ range = range.invert();
+ }
+ return range;
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
index ab525f1..5a5bcfe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundDecimalExpression.java
@@ -40,6 +40,7 @@ import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.IllegalDataException;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDecimal;
@@ -238,7 +239,11 @@ public class RoundDecimalExpression extends ScalarFunction {
throw new AssertionError("Invalid CompareOp: " + op);
}
- return KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ KeyRange range = KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ if (getColumn().getSortOrder() == SortOrder.DESC) {
+ range = range.invert();
+ }
+ return range;
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index aa9a9f5..d890383 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -774,10 +774,8 @@ public abstract class BaseResultIterators extends ExplainTable implements Result
offset = offset + rangeSpan;
}
useSkipScan &= dataScanRanges.useSkipScanFilter();
- KeyRange minMaxRange =
- clipRange(dataScanRanges.getSchema(), 0, nColumnsInCommon, dataScanRanges.getMinMaxRange());
slotSpan = slotSpan.length == cnf.size() ? slotSpan : Arrays.copyOf(slotSpan, cnf.size());
- ScanRanges commonScanRanges = ScanRanges.create(dataScanRanges.getSchema(), cnf, slotSpan, minMaxRange, null, useSkipScan, -1);
+ ScanRanges commonScanRanges = ScanRanges.create(dataScanRanges.getSchema(), cnf, slotSpan, null, useSkipScan, -1);
return commonScanRanges;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
index 265e213..1a22f60 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
@@ -258,17 +258,7 @@ public abstract class ExplainTable {
private void appendScanRow(StringBuilder buf, Bound bound) {
ScanRanges scanRanges = context.getScanRanges();
- // TODO: review this and potentially intersect the scan ranges
- // with the minMaxRange in ScanRanges to prevent having to do all this.
- KeyRange minMaxRange = scanRanges.getMinMaxRange();
Iterator<byte[]> minMaxIterator = Collections.emptyIterator();
- if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
- RowKeySchema schema = tableRef.getTable().getRowKeySchema();
- if (!minMaxRange.isUnbound(bound)) {
- // Use scan ranges from ScanRanges since it will have been intersected with minMaxRange
- minMaxIterator = new RowKeyValueIterator(schema, scanRanges.getScanRange().getRange(bound));
- }
- }
boolean isLocalIndex = ScanUtil.isLocalIndex(context.getScan());
boolean forceSkipScan = this.hint.hasHint(Hint.SKIP_SCAN);
int nRanges = forceSkipScan ? scanRanges.getRanges().size() : scanRanges.getBoundSlotCount();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
index 2159084..7d09adb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
@@ -52,12 +52,13 @@ public class KeyRange implements Writable {
public enum Bound { LOWER, UPPER };
private static final byte[] DEGENERATE_KEY = new byte[] {1};
public static final byte[] UNBOUND = new byte[0];
+ public static final byte[] NULL_BOUND = new byte[0];
/**
* KeyRange for variable length null values. Since we need to represent this using an empty byte array (which
* is what we use for upper/lower bound), we create this range using the private constructor rather than
* going through the static creation method (where this would not be possible).
*/
- public static final KeyRange IS_NULL_RANGE = new KeyRange(ByteUtil.EMPTY_BYTE_ARRAY, true, ByteUtil.EMPTY_BYTE_ARRAY, true);
+ public static final KeyRange IS_NULL_RANGE = new KeyRange(NULL_BOUND, true, NULL_BOUND, true);
/**
* KeyRange for non null variable length values. Since we need to represent this using an empty byte array (which
* is what we use for upper/lower bound), we create this range using the private constructor rather than going
@@ -131,7 +132,7 @@ public class KeyRange implements Writable {
// than an unbound RANGE.
return lowerInclusive && upperInclusive ? IS_NULL_RANGE : EVERYTHING_RANGE;
}
- if (lowerRange.length != 0 && upperRange.length != 0) {
+ if ( ( lowerRange.length != 0 || lowerRange == NULL_BOUND ) && ( upperRange.length != 0 || upperRange == NULL_BOUND ) ) {
int cmp = Bytes.compareTo(lowerRange, upperRange);
if (cmp > 0 || (cmp == 0 && !(lowerInclusive && upperInclusive))) {
return EMPTY_RANGE;
@@ -148,12 +149,12 @@ public class KeyRange implements Writable {
}
boolean unboundLower = false;
boolean unboundUpper = false;
- if (lowerRange.length == 0) {
+ if (lowerRange.length == 0 && lowerRange != NULL_BOUND) {
lowerRange = UNBOUND;
lowerInclusive = false;
unboundLower = true;
}
- if (upperRange.length == 0) {
+ if (upperRange.length == 0 && upperRange != NULL_BOUND) {
upperRange = UNBOUND;
upperInclusive = false;
unboundUpper = true;
@@ -575,20 +576,25 @@ public class KeyRange implements Writable {
}
public KeyRange invert() {
- byte[] lower = this.getLowerRange();
+ // these special ranges do not get inverted because we
+ // represent NULL in the same way for ASC and DESC.
+ if (this == IS_NOT_NULL_RANGE || this == IS_NULL_RANGE) {
+ return this;
+ }
+ byte[] lowerBound = this.getLowerRange();
if (!this.lowerUnbound()) {
- lower = SortOrder.invert(lower, 0, lower.length);
+ lowerBound = SortOrder.invert(lowerBound, 0, lowerBound.length);
}
- byte[] upper;
+ byte[] upperBound;
if (this.isSingleKey()) {
- upper = lower;
+ upperBound = lowerBound;
} else {
- upper = this.getUpperRange();
+ upperBound = this.getUpperRange();
if (!this.upperUnbound()) {
- upper = SortOrder.invert(upper, 0, upper.length);
+ upperBound = SortOrder.invert(upperBound, 0, upperBound.length);
}
}
- return KeyRange.getKeyRange(lower, this.isLowerInclusive(), upper, this.isUpperInclusive());
+ return KeyRange.getKeyRange(upperBound, this.isUpperInclusive(), lowerBound, this.isLowerInclusive());
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java
index 1a44ce1..3210516 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/RowKeySchema.java
@@ -21,7 +21,9 @@ import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.SchemaUtil;
@@ -355,4 +357,80 @@ public class RowKeySchema extends ValueSchema {
ptr.set(ptr.get(), initialOffset, finalLength);
return position + i - (Boolean.FALSE.equals(hasValue) ? 1 : 0);
}
+
+ public int computeMaxSpan(int pkPos, KeyRange result, ImmutableBytesWritable ptr) {
+ int maxOffset = iterator(result.getLowerRange(), ptr);
+ int lowerSpan = 0;
+ int i = pkPos;
+ while (this.next(ptr, i++, maxOffset) != null) {
+ lowerSpan++;
+ }
+ int upperSpan = 0;
+ i = pkPos;
+ maxOffset = iterator(result.getUpperRange(), ptr);
+ while (this.next(ptr, i++, maxOffset) != null) {
+ upperSpan++;
+ }
+ return Math.max(Math.max(lowerSpan, upperSpan), 1);
+ }
+
+ public int computeMinSpan(int pkPos, KeyRange keyRange, ImmutableBytesWritable ptr) {
+ if (keyRange == KeyRange.EVERYTHING_RANGE) {
+ return 0;
+ }
+ int lowerSpan = Integer.MAX_VALUE;
+ byte[] range = keyRange.getLowerRange();
+ if (range != KeyRange.UNBOUND) {
+ lowerSpan = 0;
+ int maxOffset = iterator(range, ptr);
+ int i = pkPos;
+ while (this.next(ptr, i++, maxOffset) != null) {
+ lowerSpan++;
+ }
+ }
+ int upperSpan = Integer.MAX_VALUE;
+ range = keyRange.getUpperRange();
+ if (range != KeyRange.UNBOUND) {
+ upperSpan = 0;
+ int maxOffset = iterator(range, ptr);
+ int i = pkPos;
+ while (this.next(ptr, i++, maxOffset) != null) {
+ upperSpan++;
+ }
+ }
+ return Math.min(lowerSpan, upperSpan);
+ }
+
+ /**
+ * Clip the left hand portion of the keyRange up to the spansToClip. If keyRange is shorter in
+ * spans than spansToClip, the portion of the range that exists will be returned.
+ * @param pkPos the leading pk position of the keyRange.
+ * @param keyRange the key range to clip
+ * @param spansToClip the number of spans to clip
+ * @param ptr an ImmutableBytesWritable to use for temporary storage.
+ * @return the clipped portion of the keyRange
+ */
+ public KeyRange clipLeft(int pkPos, KeyRange keyRange, int spansToClip, ImmutableBytesWritable ptr) {
+ if (spansToClip < 0) {
+ throw new IllegalArgumentException("Cannot specify a negative spansToClip (" + spansToClip + ")");
+ }
+ if (spansToClip == 0) {
+ return keyRange;
+ }
+ byte[] lowerRange = keyRange.getLowerRange();
+ if (lowerRange != KeyRange.UNBOUND) {
+ ptr.set(lowerRange);
+ this.position(ptr, pkPos, pkPos+spansToClip-1);
+ ptr.set(lowerRange, 0, ptr.getOffset() + ptr.getLength());
+ lowerRange = ByteUtil.copyKeyBytesIfNecessary(ptr);
+ }
+ byte[] upperRange = keyRange.getUpperRange();
+ if (upperRange != KeyRange.UNBOUND) {
+ ptr.set(upperRange);
+ this.position(ptr, pkPos, pkPos+spansToClip-1);
+ ptr.set(upperRange, 0, ptr.getOffset() + ptr.getLength());
+ upperRange = ByteUtil.copyKeyBytesIfNecessary(ptr);
+ }
+ return KeyRange.getKeyRange(lowerRange, true, upperRange, true);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index 110afc2..0ff17d3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -4500,7 +4500,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
" )\n" +
") SPLIT ON ('A','C','E','G','I')");
conn.createStatement().execute("CREATE LOCAL INDEX IDX ON T(A,B,D)");
- String query = "SELECT * FROM T WHERE A = 'C' and (A,B,D) > ('C','B','X') and D='C'";
+ String query = "SELECT * FROM T WHERE A = 'C' and (A,B,D) > ('C','B','X') and B < 'Z' and D='C'";
PhoenixStatement statement = conn.createStatement().unwrap(PhoenixStatement.class);
QueryPlan plan = statement.optimizeQuery(query);
assertEquals("IDX", plan.getContext().getCurrentTable().getTable().getName().getString());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
index 56fd178..4d5a424 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java
@@ -45,9 +45,9 @@ import org.apache.phoenix.compile.OrderByCompiler.OrderBy;
import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
import org.apache.phoenix.jdbc.PhoenixResultSet;
import org.apache.phoenix.jdbc.PhoenixStatement;
-import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.parse.DeleteStatement;
import org.apache.phoenix.parse.HintNode.Hint;
+import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PColumn;
@@ -742,9 +742,8 @@ public class QueryOptimizerTest extends BaseConnectionlessQueryTest {
assertEquals(4 + offset, plan.getContext().getScanRanges().getBoundPkColumnCount());
plan = stmt.compileQuery("select * from my_table_mt_view where pkcol1 = 'a' and pkcol2 = 'b' and pkcol3 = 'c' and (pkcol1, pkcol2) < ('z', 'z')");
assertEquals(4 + offset, plan.getContext().getScanRanges().getBoundPkColumnCount());
- // TODO: in theory pkcol2 and pkcol3 could be bound, but we don't have the logic for that yet
plan = stmt.compileQuery("select * from my_table_mt_view where (pkcol2, pkcol3) > ('0', '0') and pkcol1 = '000000000000000'");
- assertEquals(2 + offset, plan.getContext().getScanRanges().getBoundPkColumnCount());
+ assertEquals(4 + offset, plan.getContext().getScanRanges().getBoundPkColumnCount());
}
private void assertPlanDetails(PreparedStatement stmt, String expectedPkCols, String expectedPkColsDataTypes, boolean expectedHasOrderBy, int expectedLimit) throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java
index d249a66..3ab6a19 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/TenantSpecificViewIndexCompileTest.java
@@ -166,14 +166,14 @@ public class TenantSpecificViewIndexCompileTest extends BaseConnectionlessQueryT
assertOrderByHasBeenOptimizedOut(conn, sql);
// Query with predicate ordered by full row key
- sql = "SELECT * FROM v1 WHERE k3 < TO_DATE('" + createStaticDate() + "') ORDER BY k3 DESC";
- expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz','abcde',~'2015-01-01 07:59:59.999'] - ['tenant123456789','xyz','abcde',*]";
+ sql = "SELECT * FROM v1 WHERE k3 <= TO_DATE('" + createStaticDate() + "') ORDER BY k3 DESC";
+ expectedExplainOutput = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER T ['tenant123456789','xyz','abcde',~'2015-01-01 08:00:00.000'] - ['tenant123456789','xyz','abcde',*]";
assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput);
assertOrderByHasBeenOptimizedOut(conn, sql);
// Query with predicate ordered by full row key with date in reverse order
- sql = "SELECT * FROM v1 WHERE k3 < TO_DATE('" + createStaticDate() + "') ORDER BY k3";
- expectedExplainOutput = "CLIENT PARALLEL 1-WAY REVERSE RANGE SCAN OVER T ['tenant123456789','xyz','abcde',~'2015-01-01 07:59:59.999'] - ['tenant123456789','xyz','abcde',*]";
+ sql = "SELECT * FROM v1 WHERE k3 <= TO_DATE('" + createStaticDate() + "') ORDER BY k3";
+ expectedExplainOutput = "CLIENT PARALLEL 1-WAY REVERSE RANGE SCAN OVER T ['tenant123456789','xyz','abcde',~'2015-01-01 08:00:00.000'] - ['tenant123456789','xyz','abcde',*]";
assertExplainPlanIsCorrect(conn, sql, expectedExplainOutput);
assertOrderByHasBeenOptimizedOut(conn, sql);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
index 4b21a89..e5555d6 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.compile;
+import static org.apache.phoenix.query.KeyRange.EVERYTHING_RANGE;
+import static org.apache.phoenix.query.KeyRange.getKeyRange;
import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY;
import static org.apache.phoenix.util.TestUtil.BINARY_NAME;
import static org.apache.phoenix.util.TestUtil.BTABLE_NAME;
@@ -47,6 +49,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Properties;
+import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
@@ -54,7 +57,12 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.WhereOptimizer.KeyExpressionVisitor.KeySlots;
+import org.apache.phoenix.compile.WhereOptimizer.KeyExpressionVisitor.SingleKeySlot;
+import org.apache.phoenix.compile.WhereOptimizer.KeyExpressionVisitor.SlotsIterator;
+import org.apache.phoenix.compile.WhereOptimizer.KeyExpressionVisitor.TrailingRangeIterator;
import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.filter.BooleanExpressionFilter;
import org.apache.phoenix.filter.RowKeyComparisonFilter;
import org.apache.phoenix.filter.SingleCQKeyValueComparisonFilter;
import org.apache.phoenix.filter.SingleKeyValueComparisonFilter;
@@ -65,6 +73,7 @@ import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.ColumnRef;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PDate;
@@ -81,7 +90,6 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TestUtil;
-import org.apache.phoenix.schema.ColumnRef;
import org.junit.Test;
public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
@@ -109,6 +117,106 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
}
@Test
+ public void testTrailingRangesIterator() throws Exception {
+ KeyRange[] all = new KeyRange[] {EVERYTHING_RANGE,EVERYTHING_RANGE,EVERYTHING_RANGE,EVERYTHING_RANGE,EVERYTHING_RANGE, EVERYTHING_RANGE};
+ List<KeyRange[]> singleAll = Collections.singletonList(all);
+ KeyRange[] r1 = new KeyRange[] {
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ getKeyRange(Bytes.toBytes("A")),
+ EVERYTHING_RANGE, EVERYTHING_RANGE};
+ KeyRange[] r2 = new KeyRange[] {
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ getKeyRange(Bytes.toBytes("B")),
+ EVERYTHING_RANGE, EVERYTHING_RANGE};
+ KeyRange[] r3 = new KeyRange[] {
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ getKeyRange(Bytes.toBytes("C")),
+ EVERYTHING_RANGE, EVERYTHING_RANGE};
+ KeyRange[] r4 = new KeyRange[] {
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ getKeyRange(Bytes.toBytes("D")),
+ EVERYTHING_RANGE, EVERYTHING_RANGE};
+ KeyRange[] r5 = new KeyRange[] {
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ EVERYTHING_RANGE,
+ getKeyRange(Bytes.toBytes("A"),true,Bytes.toBytes("D"),true),
+ EVERYTHING_RANGE, EVERYTHING_RANGE};
+ int initPkPos = 1;
+ int pkPos = 3;
+ List<List<List<KeyRange[]>>> slotsTrailingRangesList = Lists.<List<List<KeyRange[]>>>newArrayList(
+ Lists.<List<KeyRange[]>>newArrayList(Lists.<KeyRange[]>newArrayList(r5)),
+ Lists.<List<KeyRange[]>>newArrayList(
+ Lists.<KeyRange[]>newArrayList(r1, r2),
+ Lists.<KeyRange[]>newArrayList(r3, r4)
+ ),
+ Lists.<List<KeyRange[]>>newArrayList(),
+ Lists.<List<KeyRange[]>>newArrayList(singleAll)
+ );
+ List<KeyRange> results = Lists.<KeyRange>newArrayList();
+ List<KeyRange> expectedResults = Lists.newArrayList(getKeyRange(Bytes.toBytes("A")),getKeyRange(Bytes.toBytes("B")),getKeyRange(Bytes.toBytes("C")),getKeyRange(Bytes.toBytes("D")));
+ TrailingRangeIterator iterator = new TrailingRangeIterator(initPkPos, pkPos, slotsTrailingRangesList);
+ while (iterator.hasNext()) {
+ do {
+ do {
+ KeyRange range = iterator.getRange();
+ results.add(range);
+ } while (iterator.nextTrailingRange());
+ } while (iterator.nextRange());
+ }
+ assertEquals(expectedResults, results);
+ }
+
+ @Test
+ public void testSlotsIterator() throws Exception {
+ List<KeySlots> keySlotsList = Lists.newArrayList();
+ keySlotsList.add(new SingleKeySlot(null, 0,
+ Lists.<KeyRange>newArrayList(
+ KeyRange.getKeyRange(Bytes.toBytes("A")),
+ KeyRange.getKeyRange(Bytes.toBytes("B"))
+ )));
+ keySlotsList.add(new SingleKeySlot(null, 1,
+ Lists.<KeyRange>newArrayList(
+ KeyRange.getKeyRange(Bytes.toBytes("C"))
+ )));
+ keySlotsList.add(new SingleKeySlot(null, 0,
+ Lists.<KeyRange>newArrayList(
+ KeyRange.getKeyRange(Bytes.toBytes("D")),
+ KeyRange.getKeyRange(Bytes.toBytes("E"))
+ )));
+ keySlotsList.add(new SingleKeySlot(null, 1,
+ Lists.<KeyRange>newArrayList()));
+ SlotsIterator iterator = new SlotsIterator(keySlotsList, 0);
+ String[][] expectedResults = {
+ {"A",null,"D",null},
+ {"B",null, "D", null},
+ {"A",null,"E",null},
+ {"B",null,"E",null},
+ };
+ int j = 0;
+ while (iterator.next()) {
+ int i;
+ for (i = 0; i < keySlotsList.size(); i++) {
+ KeyRange range = iterator.getRange(i);
+ String result = range == null ? null : Bytes.toString(range.getLowerRange());
+ String expectedResult = expectedResults[j][i];
+ assertEquals(expectedResult,result);
+ }
+ assertEquals(i,expectedResults[j].length);
+ j++;
+ }
+ assertEquals(j, expectedResults.length);
+ }
+
+ @Test
public void testMathFunc() throws SQLException {
Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute("create table test (id integer primary key)");
@@ -1354,20 +1462,54 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
@Test
public void testRVCExpressionThroughOr() throws SQLException {
- String tenantId = "000000000000001";
- String entityId = "002333333333331";
+ String tenantId = "000000000000001";
+ String entityId = "002333333333331";
String entityId1 = "002333333333330";
String entityId2 = "002333333333332";
String query = "select * from atable where (organization_id,entity_id) >= (?,?) and organization_id = ? and (entity_id = ? or entity_id = ?)";
List<Object> binds = Arrays.<Object>asList(tenantId, entityId, tenantId, entityId1, entityId2);
StatementContext context = compileStatement(query, binds);
Scan scan = context.getScan();
+ byte[] expectedStartRow = ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId1));
+ byte[] expectedStopRow = ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId2), QueryConstants.SEPARATOR_BYTE_ARRAY);
+ assertArrayEquals(expectedStartRow, scan.getStartRow());
+ assertArrayEquals(expectedStopRow, scan.getStopRow());
Filter filter = scan.getFilter();
- assertNull(filter);
- byte[] expectedStartRow = ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId2));
- byte[] expectedStopRow = ByteUtil.concat(ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId2)), QueryConstants.SEPARATOR_BYTE_ARRAY);
+ assertTrue(filter instanceof SkipScanFilter);
+ SkipScanFilter skipScanFilter = (SkipScanFilter)filter;
+ List<List<KeyRange>> skipScanRanges = Arrays.asList(
+ Arrays.asList(KeyRange.getKeyRange(ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId1))),
+ KeyRange.getKeyRange(ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId2)))));
+ assertEquals(skipScanRanges, skipScanFilter.getSlots());
+ }
+
+ @Test
+ public void testNotRepresentableBySkipScan() throws SQLException {
+ Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
+ String tableName = generateUniqueName();
+ conn.createStatement().execute("CREATE TABLE " + tableName + "(a INTEGER NOT NULL, b INTEGER NOT NULL, CONSTRAINT pk PRIMARY KEY (a,b))");
+ String query = "SELECT * FROM " + tableName +
+ " WHERE (a,b) >= (1,5) and (a,b) < (3,8) and (a = 1 or a = 3) and ((b >= 6 and b < 9) or (b > 3 and b <= 5))";
+ StatementContext context = compileStatement(query);
+ Scan scan = context.getScan();
+ byte[] expectedStartRow = ByteUtil.concat(PInteger.INSTANCE.toBytes(1), PInteger.INSTANCE.toBytes(4));
+ byte[] expectedStopRow = ByteUtil.concat(PInteger.INSTANCE.toBytes(3), PInteger.INSTANCE.toBytes(9));
assertArrayEquals(expectedStartRow, scan.getStartRow());
assertArrayEquals(expectedStopRow, scan.getStopRow());
+ Filter filter = scan.getFilter();
+ assertTrue(filter instanceof FilterList);
+ FilterList filterList = (FilterList)filter;
+ // We can form a skip scan, but it's not exact, so we need the boolean expression filter
+ // as well.
+ assertTrue(filterList.getFilters().get(0) instanceof SkipScanFilter);
+ assertTrue(filterList.getFilters().get(1) instanceof BooleanExpressionFilter);
+ SkipScanFilter skipScanFilter = (SkipScanFilter)filterList.getFilters().get(0);
+ List<List<KeyRange>> skipScanRanges = Arrays.asList(
+ Arrays.asList(KeyRange.getKeyRange(PInteger.INSTANCE.toBytes(1)),
+ KeyRange.getKeyRange(PInteger.INSTANCE.toBytes(3))),
+ Arrays.asList(KeyRange.getKeyRange(PInteger.INSTANCE.toBytes(4), true, PInteger.INSTANCE.toBytes(5), true),
+ KeyRange.getKeyRange(PInteger.INSTANCE.toBytes(6), true, PInteger.INSTANCE.toBytes(9), false)));
+ assertEquals(skipScanRanges, skipScanFilter.getSlots());
}
/**
@@ -1685,9 +1827,6 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
@Test
public void testQueryMoreRVC() throws SQLException {
- String tenantId = "000000000000001";
- String parentId = "000000000000008";
-
String ddl = "CREATE TABLE rvcTestIdx "
+ " (\n" +
" pk1 VARCHAR NOT NULL,\n" +
@@ -1792,9 +1931,17 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
StatementContext context = compileStatement(query, binds);
Scan scan = context.getScan();
Filter filter = scan.getFilter();
- assertTrue(filter instanceof RowKeyComparisonFilter);
+ assertTrue(filter instanceof SkipScanFilter);
assertArrayEquals(HConstants.EMPTY_START_ROW, scan.getStartRow());
assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStopRow());
+ SkipScanFilter skipScanFilter = (SkipScanFilter)filter;
+ List<List<KeyRange>> keyRanges = skipScanFilter.getSlots();
+ assertEquals(1, keyRanges.size());
+ assertEquals(2, keyRanges.get(0).size());
+ KeyRange range1 = keyRanges.get(0).get(0);
+ KeyRange range2 = keyRanges.get(0).get(1);
+ assertEquals(KeyRange.getKeyRange(KeyRange.UNBOUND, false, Bytes.toBytes(secondTenantId), true), range1);
+ assertEquals(KeyRange.getKeyRange(ByteUtil.concat(Bytes.toBytes(firstTenantId), Bytes.toBytes(firstParentId)), true, KeyRange.UNBOUND, true), range2);
}
@Test
@@ -2045,19 +2192,19 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
String query;
StatementContext context;
Connection conn = DriverManager.getConnection(getUrl());
-
+
ddl = "create table t (a integer not null, b integer not null, c integer constraint pk primary key (a,b))";
conn.createStatement().execute(ddl);
- query = "select c from t where (a,b) in ( (1,2) , (1,3) ) and b = 4";
- context = compileStatement(query, Collections.<Object>emptyList());
- assertDegenerate(context.getScan());
-
query = "select c from t where a in (1,2) and b = 3 and (a,b) in ( (1,2) , (1,3))";
context = compileStatement(query, Collections.<Object>emptyList());
assertArrayEquals(ByteUtil.concat(PInteger.INSTANCE.toBytes(1), PInteger.INSTANCE.toBytes(3)), context.getScan().getStartRow());
assertArrayEquals(ByteUtil.concat(PInteger.INSTANCE.toBytes(1), ByteUtil.nextKey(PInteger.INSTANCE.toBytes(3))), context.getScan().getStopRow());
+ query = "select c from t where (a,b) in ( (1,2) , (1,3) ) and b = 4";
+ context = compileStatement(query, Collections.<Object>emptyList());
+ assertDegenerate(context.getScan());
+
query = "select c from t where a = 1 and b = 3 and (a,b) in ( (1,2) , (1,3))";
context = compileStatement(query, Collections.<Object>emptyList());
assertArrayEquals(ByteUtil.concat(PInteger.INSTANCE.toBytes(1), PInteger.INSTANCE.toBytes(3)), context.getScan().getStartRow());
@@ -2069,8 +2216,10 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
query = "select c from t1 where d = 'a' and e = 'foo' and a in (1,2) and b = 3 and (a,b) in ( (1,2) , (1,3))";
context = compileStatement(query, Collections.<Object>emptyList());
- assertArrayEquals(ByteUtil.concat(PVarchar.INSTANCE.toBytes("a"), QueryConstants.SEPARATOR_BYTE_ARRAY, PChar.INSTANCE.toBytes("foo"), PInteger.INSTANCE.toBytes(1), PInteger.INSTANCE.toBytes(3)), context.getScan().getStartRow());
- assertArrayEquals(ByteUtil.concat(PVarchar.INSTANCE.toBytes("a"), QueryConstants.SEPARATOR_BYTE_ARRAY, PChar.INSTANCE.toBytes("foo"), PInteger.INSTANCE.toBytes(1), ByteUtil.nextKey(PInteger.INSTANCE.toBytes(3))), context.getScan().getStopRow());
+ Scan scan = context.getScan();
+ assertArrayEquals(ByteUtil.concat(PVarchar.INSTANCE.toBytes("a"), QueryConstants.SEPARATOR_BYTE_ARRAY, PChar.INSTANCE.toBytes("foo"), PInteger.INSTANCE.toBytes(1), PInteger.INSTANCE.toBytes(3)), scan.getStartRow());
+ assertArrayEquals(ByteUtil.concat(PVarchar.INSTANCE.toBytes("a"), QueryConstants.SEPARATOR_BYTE_ARRAY, PChar.INSTANCE.toBytes("foo"), PInteger.INSTANCE.toBytes(1), ByteUtil.nextKey(PInteger.INSTANCE.toBytes(3))), scan.getStopRow());
+
conn.close();
}
@@ -2092,7 +2241,7 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
}
@Test
- public void testRVCWithLeadingPKEq() throws SQLException {
+ public void testPartialRVCWithLeadingPKEq() throws SQLException {
String tenantId = "o1";
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute("CREATE TABLE COMMUNITIES.TEST (\n" +
@@ -2111,17 +2260,42 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
"AND (score, entity_id) > (2.0, '04')\n" +
"ORDER BY score, entity_id";
Scan scan = compileStatement(query).getScan();
- assertNotNull(scan.getFilter());
+ assertNull(scan.getFilter());
- // See PHOENIX-3384: Optimize RVC expressions for non leading row key columns.
- // FIXME: We should be able to optimize this better, taking into account the
- // (score, entity_id) > (2.0, '04') to form more of the start/stop row.
- assertArrayEquals(PVarchar.INSTANCE.toBytes(tenantId), scan.getStartRow());
+ byte[] startRow = ByteUtil.nextKey(ByteUtil.concat(PChar.INSTANCE.toBytes(tenantId), PDouble.INSTANCE.toBytes(2.0), PChar.INSTANCE.toBytes("04")));
+ assertArrayEquals(startRow, scan.getStartRow());
assertArrayEquals(ByteUtil.nextKey(PVarchar.INSTANCE.toBytes(tenantId)), scan.getStopRow());
}
@Test
- public void testRVCWithCompDescRowKey() throws SQLException {
+ public void testPartialRVCWithLeadingPKEqDesc() throws SQLException {
+ String tenantId = "o1";
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("CREATE TABLE COMMUNITIES.TEST (\n" +
+ " ORGANIZATION_ID CHAR(2) NOT NULL,\n" +
+ " SCORE DOUBLE NOT NULL,\n" +
+ " ENTITY_ID CHAR(2) NOT NULL\n" +
+ " CONSTRAINT PAGE_SNAPSHOT_PK PRIMARY KEY (\n" +
+ " ORGANIZATION_ID,\n" +
+ " SCORE DESC,\n" +
+ " ENTITY_ID DESC\n" +
+ " )\n" +
+ ") VERSIONS=1, MULTI_TENANT=TRUE");
+ String query = "SELECT entity_id, score\n" +
+ "FROM communities.test\n" +
+ "WHERE organization_id = '" + tenantId + "'\n" +
+ "AND (score, entity_id) < (2.0, '04')\n" +
+ "ORDER BY score DESC, entity_id DESC";
+ Scan scan = compileStatement(query).getScan();
+ assertNull(scan.getFilter());
+
+ byte[] startRow = ByteUtil.nextKey(ByteUtil.concat(PChar.INSTANCE.toBytes(tenantId), PDouble.INSTANCE.toBytes(2.0, SortOrder.DESC), PChar.INSTANCE.toBytes("04", SortOrder.DESC)));
+ assertArrayEquals(startRow, scan.getStartRow());
+ assertArrayEquals(ByteUtil.nextKey(PVarchar.INSTANCE.toBytes(tenantId)), scan.getStopRow());
+ }
+
+ @Test
+ public void testFullRVCWithLeadingPKEqDesc() throws SQLException {
String tenantId = "o1";
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute("CREATE TABLE COMMUNITIES.TEST (\n" +
@@ -2142,15 +2316,120 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
Scan scan = compileStatement(query).getScan();
assertNull(scan.getFilter());
- // FIXME See PHOENIX-3383: Comparison between descending row keys used in RVC is reverse
- // This should set the startRow, but instead it's setting the stopRow
- byte[] startRow = PChar.INSTANCE.toBytes(tenantId);
+ // TODO: end to end test that confirms this start row is accurate
+ byte[] startRow = ByteUtil.concat(PChar.INSTANCE.toBytes(tenantId), PDouble.INSTANCE.toBytes(2.0, SortOrder.DESC), ByteUtil.nextKey(PChar.INSTANCE.toBytes("04", SortOrder.DESC)));
assertArrayEquals(startRow, scan.getStartRow());
- byte[] stopRow = ByteUtil.concat(PChar.INSTANCE.toBytes(tenantId), PDouble.INSTANCE.toBytes(2.0, SortOrder.DESC), PChar.INSTANCE.toBytes("04", SortOrder.DESC));
- assertArrayEquals(stopRow, scan.getStopRow());
+ assertArrayEquals(ByteUtil.nextKey(PVarchar.INSTANCE.toBytes(tenantId)), scan.getStopRow());
+ }
+
+ @Test
+ public void testTrimTrailing() throws Exception {
+ try (Connection conn= DriverManager.getConnection(getUrl())) {
+ String sql="CREATE TABLE T("+
+ "A CHAR(1) NOT NULL,"+
+ "B CHAR(1) NOT NULL,"+
+ "C CHAR(1) NOT NULL,"+
+ "D CHAR(1) NOT NULL,"+
+ "DATA INTEGER, "+
+ "CONSTRAINT TEST_PK PRIMARY KEY (A,B,C,D))";
+ conn.createStatement().execute(sql);
+
+ // Will cause trailing part of RVC to (A,B,C) to be trimmed allowing us to perform a skip scan
+ sql="select * from T where (A,B,C) >= ('A','A','A') and (A,B,C) < ('D','D','D') and (B,C) > ('E','E')";
+ QueryPlan queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+ Scan scan = queryPlan.getContext().getScan();
+ assertTrue(scan.getFilter() instanceof SkipScanFilter);
+ List<List<KeyRange>> rowKeyRanges = ((SkipScanFilter)(scan.getFilter())).getSlots();
+ assertEquals(
+ Arrays.asList(
+ Arrays.asList(
+ KeyRange.getKeyRange(PChar.INSTANCE.toBytes("A"), true, PChar.INSTANCE.toBytes("D"), false)
+ ),
+ Arrays.asList(
+ KeyRange.getKeyRange(PChar.INSTANCE.toBytes("EE"), false, KeyRange.UNBOUND, false)
+ )
+ ),
+ rowKeyRanges
+ );
+ assertArrayEquals(scan.getStartRow(), PChar.INSTANCE.toBytes("AEF"));
+ assertArrayEquals(scan.getStopRow(), PChar.INSTANCE.toBytes("D"));
+ sql="select * from T where (A,B,C) > ('A','A','A') and (A,B,C) <= ('D','D','D') and (B,C) >= ('E','E')";
+ queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+ scan = queryPlan.getContext().getScan();
+ assertTrue(scan.getFilter() instanceof SkipScanFilter);
+ rowKeyRanges = ((SkipScanFilter)(scan.getFilter())).getSlots();
+ assertEquals(
+ Arrays.asList(
+ Arrays.asList(
+ KeyRange.getKeyRange(PChar.INSTANCE.toBytes("A"), false, PChar.INSTANCE.toBytes("D"), true)
+ ),
+ Arrays.asList(
+ KeyRange.getKeyRange(PChar.INSTANCE.toBytes("EE"), true, KeyRange.UNBOUND, false)
+ )
+ ),
+ rowKeyRanges
+ );
+ assertArrayEquals(PChar.INSTANCE.toBytes("BEE"), scan.getStartRow());
+ assertArrayEquals(PChar.INSTANCE.toBytes("E"), scan.getStopRow());
+ }
+ }
+
+ @Test
+ public void testMultiSlotTrailingIntersect() throws Exception {
+ try (Connection conn= DriverManager.getConnection(getUrl())) {
+ String sql="CREATE TABLE T("+
+ "A CHAR(1) NOT NULL,"+
+ "B CHAR(1) NOT NULL,"+
+ "C CHAR(1) NOT NULL,"+
+ "D CHAR(1) NOT NULL,"+
+ "DATA INTEGER, "+
+ "CONSTRAINT TEST_PK PRIMARY KEY (A,B,C,D))";
+ conn.createStatement().execute(sql);
+
+ sql = "select * from t where (a,b) in (('A','B'),('B','A'),('B','B'),('A','A')) and (a,b,c) in ( ('A','B','C') , ('A','C','D'), ('B','B','E'))";
+ QueryPlan queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+ Scan scan = queryPlan.getContext().getScan();
+ assertTrue(scan.getFilter() instanceof SkipScanFilter);
+ List<List<KeyRange>> rowKeyRanges = ((SkipScanFilter)(scan.getFilter())).getSlots();
+ assertEquals(
+ Arrays.asList(
+ Arrays.asList(
+ KeyRange.POINT.apply(PChar.INSTANCE.toBytes("ABC")),
+ KeyRange.POINT.apply(PChar.INSTANCE.toBytes("BBE"))
+ )
+ ),
+ rowKeyRanges
+ );
+ assertArrayEquals(scan.getStartRow(), PChar.INSTANCE.toBytes("ABC"));
+ assertArrayEquals(scan.getStopRow(), PChar.INSTANCE.toBytes("BBF"));
+ }
}
@Test
+ public void testEqualityAndGreaterThanRVC() throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute("CREATE TABLE T (\n" +
+ " A CHAR(1) NOT NULL,\n" +
+ " B CHAR(1) NOT NULL,\n" +
+ " C CHAR(1) NOT NULL,\n" +
+ " D CHAR(1) NOT NULL,\n" +
+ " CONSTRAINT PK PRIMARY KEY (\n" +
+ " A,\n" +
+ " B,\n" +
+ " C,\n" +
+ " D\n" +
+ " )\n" +
+ ")");
+ String query = "SELECT * FROM T WHERE A = 'C' and (A,B,C) > ('C','B','X') and C='C'";
+ QueryPlan queryPlan = TestUtil.getOptimizeQueryPlan(conn, query);
+ Scan scan = queryPlan.getContext().getScan();
+ assertArrayEquals(ByteUtil.concat(PChar.INSTANCE.toBytes("C"), PChar.INSTANCE.toBytes("C"), PChar.INSTANCE.toBytes("C")), scan.getStartRow());
+ assertArrayEquals(PChar.INSTANCE.toBytes("D"), scan.getStopRow());
+ }
+ }
+
+ @Test
public void testOrExpressionNonLeadingPKPushToScanBug4602() throws Exception {
Connection conn = null;
try {
@@ -2163,7 +2442,7 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
"DATA INTEGER, "+
"CONSTRAINT TEST_PK PRIMARY KEY (PK1,PK2,PK3))";
conn.createStatement().execute(sql);
-
+
//case 1: pk1 is equal,pk2 is multiRange
sql="select * from "+testTableName+" t where (t.pk1 = 2) and ((t.pk2 >= 4 and t.pk2 <6) or (t.pk2 >= 8 and t.pk2 <9))";
QueryPlan queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
@@ -2302,8 +2581,8 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
sql ="select * from "+testTableName+" t where (t.pk1 >=2 and t.pk1<5) or (t.pk2 >=7 or t.pk2 <9)";
queryPlan= TestUtil.getOptimizeQueryPlan(conn, sql);
- Expression pk2Expression = new ColumnRef(queryPlan.getTableRef(), queryPlan.getTableRef().getTable().getColumnForColumnName("PK2").getPosition()).newColumnExpression();
scan = queryPlan.getContext().getScan();
+ Expression pk2Expression = new ColumnRef(queryPlan.getTableRef(), queryPlan.getTableRef().getTable().getColumnForColumnName("PK2").getPosition()).newColumnExpression();
assertTrue(scan.getFilter() instanceof RowKeyComparisonFilter);
assertEquals(
TestUtil.rowKeyFilter(
@@ -2372,6 +2651,24 @@ public class WhereOptimizerTest extends BaseConnectionlessQueryTest {
scan.getFilter());
assertArrayEquals(scan.getStartRow(), HConstants.EMPTY_START_ROW);
assertArrayEquals(scan.getStopRow(), HConstants.EMPTY_END_ROW);
+
+ //case 11: pk1 and pk2, but pk1 has a or allRange and force skip scan
+ sql="select /*+ SKIP_SCAN */ * from "+testTableName+" t where ((t.pk1 >=2 and t.pk1<5) or (t.pk1 >=7 or t.pk1 <9)) and ((t.pk2 >= 4 and t.pk2 <6) or (t.pk2 >= 8 and t.pk2 <9))";
+ queryPlan = TestUtil.getOptimizeQueryPlan(conn, sql);
+ scan = queryPlan.getContext().getScan();
+ assertTrue(scan.getFilter() instanceof SkipScanFilter);
+ rowKeyRanges = ((SkipScanFilter)(scan.getFilter())).getSlots();
+ assertEquals(
+ Arrays.asList(
+ Arrays.asList(KeyRange.EVERYTHING_RANGE),
+ Arrays.asList(
+ KeyRange.getKeyRange(PInteger.INSTANCE.toBytes(4), true, PInteger.INSTANCE.toBytes(6), false),
+ KeyRange.getKeyRange(PInteger.INSTANCE.toBytes(8), true, PInteger.INSTANCE.toBytes(9), false)
+ )
+ ),
+ rowKeyRanges);
+ assertArrayEquals(scan.getStartRow(), HConstants.EMPTY_START_ROW);
+ assertArrayEquals(scan.getStopRow(), HConstants.EMPTY_END_ROW);
}
finally {
if(conn!=null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java
index 89058ba..3562099 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/RoundFloorCeilExpressionsTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.expression;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -24,6 +25,7 @@ import static org.junit.Assert.fail;
import java.math.BigDecimal;
import java.sql.Date;
+import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -42,14 +44,20 @@ import org.apache.phoenix.expression.function.RoundDecimalExpression;
import org.apache.phoenix.expression.function.ScalarFunction;
import org.apache.phoenix.expression.function.TimeUnit;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.IllegalDataException;
+import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.schema.types.PDecimal;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Test;
/**
@@ -60,7 +68,7 @@ import org.junit.Test;
*
* @since 3.0.0
*/
-public class RoundFloorCeilExpressionsTest {
+public class RoundFloorCeilExpressionsTest extends BaseConnectionlessQueryTest {
// Decimal Expression Tests
@@ -165,37 +173,40 @@ public class RoundFloorCeilExpressionsTest {
@Test
public void testRoundDecimalExpressionKeyRangeSimple() throws Exception {
+ KeyPart baseKeyPart = getDecimalKeyPart();
ScalarFunction roundDecimalExpression = (ScalarFunction)RoundDecimalExpression.create(DUMMY_DECIMAL, 3);
byte[] upperBound = PDecimal.INSTANCE.toBytes(new BigDecimal("1.2385"));
byte[] lowerBound = PDecimal.INSTANCE.toBytes(new BigDecimal("1.2375"));
KeyRange expectedKeyRange = KeyRange.getKeyRange(lowerBound, upperBound);
- KeyPart keyPart = roundDecimalExpression.newKeyPart(null);
+ KeyPart keyPart = roundDecimalExpression.newKeyPart(baseKeyPart);
assertEquals(expectedKeyRange, keyPart.getKeyRange(CompareOp.EQUAL, LiteralExpression.newConstant(new BigDecimal("1.238"), PDecimal.INSTANCE)));
}
@Test
public void testFloorDecimalExpressionKeyRangeSimple() throws Exception {
+ KeyPart baseKeyPart = getDecimalKeyPart();
ScalarFunction floorDecimalExpression = (ScalarFunction)FloorDecimalExpression.create(DUMMY_DECIMAL, 3);
byte[] upperBound = PDecimal.INSTANCE.toBytes(new BigDecimal("1.239"));
byte[] lowerBound = PDecimal.INSTANCE.toBytes(new BigDecimal("1.238"));
KeyRange expectedKeyRange = KeyRange.getKeyRange(lowerBound, true, upperBound, false);
- KeyPart keyPart = floorDecimalExpression.newKeyPart(null);
+ KeyPart keyPart = floorDecimalExpression.newKeyPart(baseKeyPart);
assertEquals(expectedKeyRange, keyPart.getKeyRange(CompareOp.EQUAL, LiteralExpression.newConstant(new BigDecimal("1.238"), PDecimal.INSTANCE)));
}
@Test
public void testCeilDecimalExpressionKeyRangeSimple() throws Exception {
+ KeyPart baseKeyPart = getDecimalKeyPart();
ScalarFunction ceilDecimalExpression = (ScalarFunction)CeilDecimalExpression.create(DUMMY_DECIMAL, 3);
byte[] upperBound = PDecimal.INSTANCE.toBytes(new BigDecimal("1.238"));
byte[] lowerBound = PDecimal.INSTANCE.toBytes(new BigDecimal("1.237"));
KeyRange expectedKeyRange = KeyRange.getKeyRange(lowerBound, false, upperBound, true);
- KeyPart keyPart = ceilDecimalExpression.newKeyPart(null);
+ KeyPart keyPart = ceilDecimalExpression.newKeyPart(baseKeyPart);
assertEquals(expectedKeyRange, keyPart.getKeyRange(CompareOp.EQUAL, LiteralExpression.newConstant(new BigDecimal("1.238"), PDecimal.INSTANCE)));
}
@@ -203,27 +214,61 @@ public class RoundFloorCeilExpressionsTest {
@Test
public void testRoundDecimalExpressionKeyRangeCoverage() throws Exception {
+ KeyPart baseKeyPart = getDecimalKeyPart();
for(int scale : SCALES) {
ScalarFunction roundDecimalExpression = (ScalarFunction) RoundDecimalExpression.create(DUMMY_DECIMAL, scale);
- KeyPart keyPart = roundDecimalExpression.newKeyPart(null);
+ KeyPart keyPart = roundDecimalExpression.newKeyPart(baseKeyPart);
verifyKeyPart(RoundingType.ROUND, scale, keyPart);
}
}
+ private static KeyPart getDecimalKeyPart() throws SQLException {
+ String tableName = generateUniqueName();
+ try (PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class)) {
+ pconn.createStatement().execute("CREATE TABLE " + tableName + " (k DECIMAL PRIMARY KEY)");
+ final PTable table = pconn.getMetaDataCache().getTableRef(new PTableKey(null, tableName)).getTable();
+ KeyPart baseKeyPart = new KeyPart() {
+
+ @Override
+ public KeyRange getKeyRange(CompareOp op, Expression rhs) {
+ return KeyRange.EVERYTHING_RANGE;
+ }
+
+ @Override
+ public List<Expression> getExtractNodes() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public PColumn getColumn() {
+ return table.getPKColumns().get(0);
+ }
+
+ @Override
+ public PTable getTable() {
+ return table;
+ }
+ };
+ return baseKeyPart;
+ }
+ }
+
@Test
public void testFloorDecimalExpressionKeyRangeCoverage() throws Exception {
+ KeyPart baseKeyPart = getDecimalKeyPart();
for(int scale : SCALES) {
ScalarFunction floorDecimalExpression = (ScalarFunction) FloorDecimalExpression.create(DUMMY_DECIMAL, scale);
- KeyPart keyPart = floorDecimalExpression.newKeyPart(null);
+ KeyPart keyPart = floorDecimalExpression.newKeyPart(baseKeyPart);
verifyKeyPart(RoundingType.FLOOR, scale, keyPart);
}
}
@Test
public void testCeilDecimalExpressionKeyRangeCoverage() throws Exception {
+ KeyPart baseKeyPart = getDecimalKeyPart();
for(int scale : SCALES) {
ScalarFunction ceilDecimalExpression = (ScalarFunction) CeilDecimalExpression.create(DUMMY_DECIMAL, scale);
- KeyPart keyPart = ceilDecimalExpression.newKeyPart(null);
+ KeyPart keyPart = ceilDecimalExpression.newKeyPart(baseKeyPart);
verifyKeyPart(RoundingType.CEIL, scale, keyPart);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/query/KeyRangeClipTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/KeyRangeClipTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/KeyRangeClipTest.java
index abc435a..a3da784 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/KeyRangeClipTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/KeyRangeClipTest.java
@@ -104,7 +104,7 @@ public class KeyRangeClipTest extends BaseConnectionlessQueryTest {
@Test
public void test() {
- ScanRanges scanRanges = ScanRanges.create(schema, Collections.<List<KeyRange>>singletonList(Collections.<KeyRange>singletonList(input)), new int[] {schema.getFieldCount()-1}, KeyRange.EVERYTHING_RANGE, null, false, -1);
+ ScanRanges scanRanges = ScanRanges.create(schema, Collections.<List<KeyRange>>singletonList(Collections.<KeyRange>singletonList(input)), new int[] {schema.getFieldCount()-1}, null, false, -1);
ScanRanges clippedRange = BaseResultIterators.computePrefixScanRanges(scanRanges, clipTo);
assertEquals(expectedOutput, clippedRange.getScanRange());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java
index 787ca33..d87989c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryPlanTest.java
@@ -36,6 +36,9 @@ public class QueryPlanTest extends BaseConnectionlessQueryTest {
public void testExplainPlan() throws Exception {
String[] queryPlans = new String[] {
+ "SELECT a_string,b_string FROM atable WHERE organization_id = '000000000000001' AND entity_id > '000000000000002' AND entity_id < '000000000000008' AND (organization_id,entity_id) <= ('000000000000001','000000000000005') ",
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER ATABLE ['000000000000001','000000000000003'] - ['000000000000001','000000000000005']",
+
"SELECT host FROM PTSDB WHERE inst IS NULL AND host IS NOT NULL AND \"DATE\" >= to_date('2013-01-01')",
"CLIENT PARALLEL 1-WAY RANGE SCAN OVER PTSDB [null,not null]\n" +
" SERVER FILTER BY FIRST KEY ONLY AND \"DATE\" >= DATE '2013-01-01 00:00:00.000'",
@@ -64,11 +67,8 @@ public class QueryPlanTest extends BaseConnectionlessQueryTest {
"CLIENT PARALLEL 1-WAY POINT LOOKUP ON 1 KEY OVER ATABLE\n" +
" SERVER FILTER BY (X_INTEGER = 2 AND A_INTEGER < 5)",
- "SELECT a_string,b_string FROM atable WHERE organization_id = '000000000000001' AND entity_id > '000000000000002' AND entity_id < '000000000000008' AND (organization_id,entity_id) <= ('000000000000001','000000000000005') ",
- "CLIENT PARALLEL 1-WAY RANGE SCAN OVER ATABLE ['000000000000001','000000000000003'] - ['000000000000001','000000000000006']",
-
"SELECT a_string,b_string FROM atable WHERE organization_id > '000000000000001' AND entity_id > '000000000000002' AND entity_id < '000000000000008' AND (organization_id,entity_id) >= ('000000000000003','000000000000005') ",
- "CLIENT PARALLEL 1-WAY RANGE SCAN OVER ATABLE ['000000000000003','000000000000005'] - [*]\n" +
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER ATABLE ['000000000000003000000000000005'] - [*]\n" +
" SERVER FILTER BY (ENTITY_ID > '000000000000002' AND ENTITY_ID < '000000000000008')",
"SELECT a_string,b_string FROM atable WHERE organization_id = '000000000000001' AND entity_id >= '000000000000002' AND entity_id < '000000000000008' AND (organization_id,entity_id) >= ('000000000000000','000000000000005') ",
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/test/java/org/apache/phoenix/schema/RowKeySchemaTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/RowKeySchemaTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/RowKeySchemaTest.java
index a435ba6..35278a7 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/schema/RowKeySchemaTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/RowKeySchemaTest.java
@@ -25,15 +25,19 @@ import static org.junit.Assert.assertTrue;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.sql.SQLException;
import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.Test;
@@ -148,4 +152,48 @@ public class RowKeySchemaTest extends BaseConnectionlessQueryTest {
assertExpectedRowKeyValue("c1 VARCHAR, c2 CHAR(1) NOT NULL, c3 INTEGER NOT NULL", "c1, c2, c3", new Object[] {"abc", "z", 5});
}
+ private static byte[] getKeyPart(PTable t, String... keys) throws SQLException {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ byte[][] keyByteArray = new byte[keys.length][];
+ int i = 0;
+ for (String key : keys) {
+ keyByteArray[i++] = key == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(key);
+ }
+ t.newKey(ptr, keyByteArray);
+ return ptr.copyBytes();
+ }
+
+ @Test
+ public void testClipLeft() throws Exception {
+ ImmutableBytesWritable ptr = new ImmutableBytesWritable();
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("CREATE TABLE T1(K1 CHAR(1) NOT NULL, K2 VARCHAR, K3 VARCHAR, CONSTRAINT pk PRIMARY KEY (K1,K2,K3)) ");
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table;
+ RowKeySchema schema;
+ table = pconn.getTable(new PTableKey(pconn.getTenantId(), "T1"));
+ schema = table.getRowKeySchema();
+ KeyRange r, rLeft, expectedResult;
+ r = KeyRange.getKeyRange(getKeyPart(table, "A", "B", "C"), true, getKeyPart(table, "B", "C"), true);
+ rLeft = schema.clipLeft(0, r, 1, ptr);
+ expectedResult = KeyRange.getKeyRange(getKeyPart(table, "A"), true, getKeyPart(table, "B"), true);
+ r = KeyRange.getKeyRange(getKeyPart(table, "A", "B", "C"), true, getKeyPart(table, "B"), true);
+ rLeft = schema.clipLeft(0, r, 1, ptr);
+ expectedResult = KeyRange.getKeyRange(getKeyPart(table, "A"), true, getKeyPart(table, "B"), true);
+ assertEquals(expectedResult, rLeft);
+ rLeft = schema.clipLeft(0, r, 2, ptr);
+ expectedResult = KeyRange.getKeyRange(getKeyPart(table, "A", "B"), true, getKeyPart(table, "B"), true);
+ assertEquals(expectedResult, rLeft);
+
+ r = KeyRange.getKeyRange(getKeyPart(table, "A", "B", "C"), true, KeyRange.UNBOUND, true);
+ rLeft = schema.clipLeft(0, r, 2, ptr);
+ expectedResult = KeyRange.getKeyRange(getKeyPart(table, "A", "B"), true, KeyRange.UNBOUND, false);
+ assertEquals(expectedResult, rLeft);
+
+ r = KeyRange.getKeyRange(KeyRange.UNBOUND, false, getKeyPart(table, "A", "B", "C"), true);
+ rLeft = schema.clipLeft(0, r, 2, ptr);
+ expectedResult = KeyRange.getKeyRange(KeyRange.UNBOUND, false, getKeyPart(table, "A", "B"), true);
+ assertEquals(expectedResult, rLeft);
+ }
+
}
[2/4] phoenix git commit: PHOENIX-3383 Comparison between descending
row keys used in RVC is reverse
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
index 87f00e4..a5287cb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/WhereOptimizer.java
@@ -17,8 +17,6 @@
*/
package org.apache.phoenix.compile;
-import static java.util.Collections.singletonList;
-
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -27,10 +25,14 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Set;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.expression.AndExpression;
import org.apache.phoenix.expression.BaseExpression;
import org.apache.phoenix.expression.BaseExpression.ExpressionComparabilityWrapper;
@@ -61,7 +63,6 @@ import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.SortOrder;
-import org.apache.phoenix.schema.ValueSchema.Field;
import org.apache.phoenix.schema.tuple.Tuple;
import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PDataType;
@@ -74,8 +75,11 @@ import org.apache.phoenix.util.SchemaUtil;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import edu.umd.cs.findbugs.annotations.NonNull;
+
/**
*
@@ -115,6 +119,7 @@ public class WhereOptimizer {
RowKeySchema schema = table.getRowKeySchema();
boolean isMultiTenant = tenantId != null && table.isMultiTenant();
boolean isSharedIndex = table.getViewIndexId() != null;
+ ImmutableBytesWritable ptr = context.getTempPtr();
if (isMultiTenant) {
tenantIdBytes = ScanUtil.getTenantIdBytes(schema, isSalted, tenantId, isSharedIndex);
@@ -158,31 +163,13 @@ public class WhereOptimizer {
int pkPos = 0;
int nPKColumns = table.getPKColumns().size();
- int[] slotSpan = new int[nPKColumns];
+ int[] slotSpanArray = new int[nPKColumns];
List<List<KeyRange>> cnf = Lists.newArrayListWithExpectedSize(schema.getMaxFields());
- KeyRange minMaxRange = keySlots.getMinMaxRange();
- if (minMaxRange == null) {
- minMaxRange = KeyRange.EVERYTHING_RANGE;
- }
- boolean hasMinMaxRange = (minMaxRange != KeyRange.EVERYTHING_RANGE);
- int minMaxRangeOffset = 0;
- byte[] minMaxRangePrefix = null;
boolean hasViewIndex = table.getViewIndexId() != null;
- if (hasMinMaxRange) {
- int minMaxRangeSize = (isSalted ? SaltingUtil.NUM_SALTING_BYTES : 0)
- + (isMultiTenant ? tenantIdBytes.length + 1 : 0)
- + (hasViewIndex ? MetaDataUtil.getViewIndexIdDataType().getByteSize() : 0);
- minMaxRangePrefix = new byte[minMaxRangeSize];
- }
-
- Iterator<KeyExpressionVisitor.KeySlot> iterator = keySlots.iterator();
+ Iterator<KeyExpressionVisitor.KeySlot> iterator = keySlots.getSlots().iterator();
// Add placeholder for salt byte ranges
if (isSalted) {
cnf.add(SALT_PLACEHOLDER);
- if (hasMinMaxRange) {
- System.arraycopy(SALT_PLACEHOLDER.get(0).getLowerRange(), 0, minMaxRangePrefix, minMaxRangeOffset, SaltingUtil.NUM_SALTING_BYTES);
- minMaxRangeOffset += SaltingUtil.NUM_SALTING_BYTES;
- }
// Increment the pkPos, as the salt column is in the row schema
// Do not increment the iterator, though, as there will never be
// an expression in the keySlots for the salt column
@@ -194,35 +181,17 @@ public class WhereOptimizer {
if (hasViewIndex) {
byte[] viewIndexBytes = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
KeyRange indexIdKeyRange = KeyRange.getKeyRange(viewIndexBytes);
- cnf.add(singletonList(indexIdKeyRange));
- if (hasMinMaxRange) {
- System.arraycopy(viewIndexBytes, 0, minMaxRangePrefix, minMaxRangeOffset, viewIndexBytes.length);
- minMaxRangeOffset += viewIndexBytes.length;
- }
+ cnf.add(Collections.singletonList(indexIdKeyRange));
pkPos++;
}
// Add tenant data isolation for tenant-specific tables
if (isMultiTenant) {
KeyRange tenantIdKeyRange = KeyRange.getKeyRange(tenantIdBytes);
- cnf.add(singletonList(tenantIdKeyRange));
- if (hasMinMaxRange) {
- System.arraycopy(tenantIdBytes, 0, minMaxRangePrefix, minMaxRangeOffset, tenantIdBytes.length);
- minMaxRangeOffset += tenantIdBytes.length;
- Field f = schema.getField(pkPos);
- if (!f.getDataType().isFixedWidth()) {
- minMaxRangePrefix[minMaxRangeOffset] = SchemaUtil.getSeparatorByte(schema.rowKeyOrderOptimizable(), tenantIdBytes.length==0, f);
- minMaxRangeOffset++;
- }
- }
+ cnf.add(Collections.singletonList(tenantIdKeyRange));
pkPos++;
}
- // Prepend minMaxRange with fixed column values so we can properly intersect the
- // range with the other range.
- if (hasMinMaxRange) {
- minMaxRange = minMaxRange.prependRange(minMaxRangePrefix, 0, minMaxRangeOffset);
- }
boolean forcedSkipScan = statement.getHint().hasHint(Hint.SKIP_SCAN);
boolean forcedRangeScan = statement.getHint().hasHint(Hint.RANGE_SCAN);
boolean hasUnboundedRange = false;
@@ -251,13 +220,44 @@ public class WhereOptimizer {
}
}
KeyPart keyPart = slot.getKeyPart();
- slotSpan[cnf.size()] = slot.getPKSpan() - 1;
- pkPos = slot.getPKPosition() + slot.getPKSpan();
- // Skip span-1 slots as we skip one at the top of the loop
- for (int i = 1; i < slot.getPKSpan() && iterator.hasNext(); i++) {
- iterator.next();
- }
List<KeyRange> keyRanges = slot.getKeyRanges();
+ SortOrder prevSortOrder = null;
+ int slotOffset = 0;
+ int clipLeftSpan = 0;
+
+ // Iterate through all spans of this slot
+ while (true) {
+ SortOrder sortOrder = schema.getField(slot.getPKPosition() + slotOffset).getSortOrder();
+ if (prevSortOrder == null) {
+ prevSortOrder = sortOrder;
+ } else if (prevSortOrder != sortOrder) {
+ // If the sort order changes, we must clip the portion with the same sort order
+ // and invert the key ranges and swap the upper and lower bounds.
+ List<KeyRange> leftRanges = clipLeft(schema, slot.getPKPosition() + slotOffset - clipLeftSpan, clipLeftSpan, keyRanges, ptr);
+ keyRanges = clipRight(schema, slot.getPKPosition() + slotOffset - 1, keyRanges, leftRanges, ptr);
+ if (prevSortOrder == SortOrder.DESC) {
+ leftRanges = invertKeyRanges(leftRanges);
+ }
+ slotSpanArray[cnf.size()] = clipLeftSpan-1;
+ cnf.add(leftRanges);
+ clipLeftSpan = 0;
+ prevSortOrder = sortOrder;
+ }
+ clipLeftSpan++;
+ slotOffset++;
+ if (slotOffset >= slot.getPKSpan()) {
+ break;
+ }
+ if (iterator.hasNext()) {
+ iterator.next();
+ }
+ }
+ if (schema.getField(slot.getPKPosition() + slotOffset - 1).getSortOrder() == SortOrder.DESC) {
+ keyRanges = invertKeyRanges(keyRanges);
+ }
+ pkPos = slot.getPKPosition() + slot.getPKSpan();
+
+ slotSpanArray[cnf.size()] = clipLeftSpan-1;
cnf.add(keyRanges);
// TODO: when stats are available, we may want to use a skip scan if the
@@ -301,8 +301,8 @@ public class WhereOptimizer {
// If we have fully qualified point keys with multi-column spans (i.e. RVC),
// we can still use our skip scan. The ScanRanges.create() call will explode
// out the keys.
- slotSpan = Arrays.copyOf(slotSpan, cnf.size());
- ScanRanges scanRanges = ScanRanges.create(schema, cnf, slotSpan, minMaxRange, nBuckets, useSkipScan, table.getRowTimestampColPos());
+ slotSpanArray = Arrays.copyOf(slotSpanArray, cnf.size());
+ ScanRanges scanRanges = ScanRanges.create(schema, cnf, slotSpanArray, nBuckets, useSkipScan, table.getRowTimestampColPos());
context.setScanRanges(scanRanges);
if (whereClause == null) {
return null;
@@ -311,6 +311,61 @@ public class WhereOptimizer {
}
}
+ private static KeyRange getTrailingRange(RowKeySchema rowKeySchema, int pkPos, KeyRange range, KeyRange clippedResult, ImmutableBytesWritable ptr) {
+ int separatorLength = rowKeySchema.getField(pkPos).getDataType().isFixedWidth() ? 0 : 1;
+ byte[] lowerRange = KeyRange.UNBOUND;
+ boolean lowerInclusive = false;
+ // Lower range of trailing part of RVC must be true, so we can form a new range to intersect going forward
+ if (!range.lowerUnbound() && Bytes.startsWith(range.getLowerRange(), clippedResult.getLowerRange())) {
+ lowerRange = range.getLowerRange();
+ int offset = clippedResult.getLowerRange().length + separatorLength;
+ ptr.set(lowerRange, offset, lowerRange.length - offset);
+ lowerRange = ptr.copyBytes();
+ lowerInclusive = range.isLowerInclusive();
+ }
+ byte[] upperRange = KeyRange.UNBOUND;
+ boolean upperInclusive = false;
+ if (!range.upperUnbound() && Bytes.startsWith(range.getUpperRange(), clippedResult.getUpperRange())) {
+ upperRange = range.getUpperRange();
+ int offset = clippedResult.getUpperRange().length + separatorLength;
+ ptr.set(upperRange, offset, upperRange.length - offset);
+ upperRange = ptr.copyBytes();
+ upperInclusive = range.isUpperInclusive();
+ }
+ return KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ }
+
+ private static List<KeyRange> clipRight(RowKeySchema schema, int pkPos, List<KeyRange> keyRanges,
+ List<KeyRange> leftRanges, ImmutableBytesWritable ptr) {
+ List<KeyRange> clippedKeyRanges = Lists.newArrayListWithExpectedSize(keyRanges.size());
+ for (int i = 0; i < leftRanges.size(); i++) {
+ KeyRange leftRange = leftRanges.get(i);
+ KeyRange range = keyRanges.get(i);
+ KeyRange clippedKeyRange = getTrailingRange(schema, pkPos, range, leftRange, ptr);
+ clippedKeyRanges.add(clippedKeyRange);
+ }
+ return clippedKeyRanges;
+ }
+
+ private static List<KeyRange> clipLeft(RowKeySchema schema, int pkPos, int clipLeftSpan, List<KeyRange> keyRanges, ImmutableBytesWritable ptr) {
+ List<KeyRange> clippedKeyRanges = Lists.newArrayListWithExpectedSize(keyRanges.size());
+ for (KeyRange keyRange : keyRanges) {
+ KeyRange clippedKeyRange = schema.clipLeft(pkPos, keyRange, clipLeftSpan, ptr);
+ clippedKeyRanges.add(clippedKeyRange);
+ }
+ return clippedKeyRanges;
+ }
+
+ private static List<KeyRange> invertKeyRanges(List<KeyRange> keyRanges) {
+ keyRanges = new ArrayList<KeyRange>(keyRanges);
+ for (int i = 0; i < keyRanges.size(); i++) {
+ KeyRange range = keyRanges.get(i);
+ range = range.invert();
+ keyRanges.set(i, range);
+ }
+ return keyRanges;
+ }
+
/**
* Get an optimal combination of key expressions for hash join key range optimization.
* @return returns true if the entire combined expression is covered by key range optimization
@@ -330,7 +385,7 @@ public class WhereOptimizer {
KeyExpressionVisitor.KeySlots keySlots = expression.accept(visitor);
int minPkPos = Integer.MAX_VALUE;
if (keySlots != null) {
- Iterator<KeyExpressionVisitor.KeySlot> iterator = keySlots.iterator();
+ Iterator<KeyExpressionVisitor.KeySlot> iterator = keySlots.getSlots().iterator();
while (iterator.hasNext()) {
KeyExpressionVisitor.KeySlot slot = iterator.next();
if (slot.getPKPosition() < minPkPos) {
@@ -457,18 +512,13 @@ public class WhereOptimizer {
public static class KeyExpressionVisitor extends StatelessTraverseNoExpressionVisitor<KeyExpressionVisitor.KeySlots> {
private static final KeySlots EMPTY_KEY_SLOTS = new KeySlots() {
@Override
- public Iterator<KeySlot> iterator() {
- return Collections.emptyIterator();
- }
-
- @Override
- public KeyRange getMinMaxRange() {
- return null;
+ public boolean isPartialExtraction() {
+ return false;
}
@Override
- public boolean isPartialExtraction() {
- return false;
+ public List<KeySlot> getSlots() {
+ return Collections.emptyList();
}
};
@@ -481,22 +531,11 @@ public class WhereOptimizer {
return EMPTY_KEY_SLOTS;
}
- List<KeyRange> keyRanges = slot.getPKSpan() == 1 ? Collections.<KeyRange>singletonList(keyRange) : EVERYTHING_RANGES;
- KeyRange minMaxRange = null;
- if (slot.getPKSpan() > 1) {
- int initPosition = (table.getBucketNum() ==null ? 0 : 1) + (this.context.getConnection().getTenantId() != null && table.isMultiTenant() ? 1 : 0) + (table.getViewIndexId() == null ? 0 : 1);
- if (initPosition == slot.getPKPosition()) {
- minMaxRange = keyRange;
- } else {
- // If we cannot set the minMaxRange, then we must not extract the expression since
- // we wouldn't be constraining the range at all based on it.
- extractNode = null;
- }
- }
- return newKeyParts(slot, extractNode, keyRanges, minMaxRange);
+ List<KeyRange> keyRanges = Collections.<KeyRange>singletonList(keyRange);
+ return newKeyParts(slot, extractNode, keyRanges);
}
- private KeySlots newKeyParts(KeySlot slot, Expression extractNode, List<KeyRange> keyRanges, KeyRange minMaxRange) {
+ private KeySlots newKeyParts(KeySlot slot, Expression extractNode, List<KeyRange> keyRanges) {
if (isDegenerate(keyRanges)) {
return EMPTY_KEY_SLOTS;
}
@@ -504,15 +543,15 @@ public class WhereOptimizer {
List<Expression> extractNodes = extractNode == null || slot.getKeyPart().getExtractNodes().isEmpty()
? Collections.<Expression>emptyList()
: Collections.<Expression>singletonList(extractNode);
- return new SingleKeySlot(new BaseKeyPart(table, slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, minMaxRange, slot.getOrderPreserving());
+ return new SingleKeySlot(new BaseKeyPart(table, slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, slot.getOrderPreserving());
}
- private KeySlots newKeyParts(KeySlot slot, List<Expression> extractNodes, List<KeyRange> keyRanges, KeyRange minMaxRange) {
+ private KeySlots newKeyParts(KeySlot slot, List<Expression> extractNodes, List<KeyRange> keyRanges) {
if (isDegenerate(keyRanges)) {
return EMPTY_KEY_SLOTS;
}
- return new SingleKeySlot(new BaseKeyPart(table, slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, minMaxRange, slot.getOrderPreserving());
+ return new SingleKeySlot(new BaseKeyPart(table, slot.getKeyPart().getColumn(), extractNodes), slot.getPKPosition(), slot.getPKSpan(), keyRanges, slot.getOrderPreserving());
}
private KeySlots newRowValueConstructorKeyParts(RowValueConstructorExpression rvc, List<KeySlots> childSlots) {
@@ -524,7 +563,7 @@ public class WhereOptimizer {
int initialPosition = -1;
for (int i = 0; i < childSlots.size(); i++) {
KeySlots slots = childSlots.get(i);
- KeySlot keySlot = slots.iterator().next();
+ KeySlot keySlot = slots.getSlots().iterator().next();
List<Expression> childExtractNodes = keySlot.getKeyPart().getExtractNodes();
// Stop if there was a gap in extraction of RVC elements. This is required if the leading
// RVC has not row key columns, as we'll still get childSlots if the RVC has trailing row
@@ -594,17 +633,18 @@ public class WhereOptimizer {
// For the actual type of the coerceBytes call, we use the node type instead of the rhs type, because
// for IN, the rhs type will be VARBINARY and no coerce will be done in that case (and we need it to
// be done).
- node.getChild().getDataType().coerceBytes(ptr, node.getDataType(), rhs.getSortOrder(), node.getChild().getSortOrder());
+ node.getChild().getDataType().coerceBytes(ptr, node.getDataType(), rhs.getSortOrder(), SortOrder.ASC);
lower = ByteUtil.copyKeyBytesIfNecessary(ptr);
}
byte[] upper = range.getUpperRange();
if (!range.upperUnbound()) {
ptr.set(upper);
// Do the reverse translation so we can optimize out the coerce expression
- node.getChild().getDataType().coerceBytes(ptr, node.getDataType(), rhs.getSortOrder(), node.getChild().getSortOrder());
+ node.getChild().getDataType().coerceBytes(ptr, node.getDataType(), rhs.getSortOrder(), SortOrder.ASC);
upper = ByteUtil.copyKeyBytesIfNecessary(ptr);
}
- return KeyRange.getKeyRange(lower, range.isLowerInclusive(), upper, range.isUpperInclusive());
+ range = KeyRange.getKeyRange(lower, range.isLowerInclusive(), upper, range.isUpperInclusive());
+ return range;
}
@Override
@@ -624,94 +664,739 @@ public class WhereOptimizer {
}, slot.getPKPosition(), slot.getKeyRanges());
}
- private static boolean intersectSlots(KeySlot[] slotArray, KeySlot childSlot) {
- int childPosition = childSlot.getPKPosition();
- int childSpan = childSlot.getPKSpan();
- boolean filled = false;
- for (KeySlot slot : slotArray) {
- if (slot != null) {
- int position = slot.getPKPosition();
- int span = slot.getPKSpan();
- if (childPosition + childSpan > position && childPosition < position + span) {
- if (childPosition < position || (childPosition == position && childSpan > span)) {
- slotArray[childPosition] = childSlot = childSlot.intersect(slot);
- if (childSlot == null) {
- return false;
- }
- filled = true;
- } else {
- slotArray[position] = slot = slot.intersect(childSlot);
- if (slot == null) {
+ /**
+ *
+ * Iterates through all combinations of KeyRanges for a given
+ * PK column (based on its slot position). Useful when expressions
+ * are ORed together and subsequently ANDed. For example:
+ * WHERE (pk1 = 1 OR pk1 = 2) AND (pk2 = 3 OR pk2 = 4)
+ * would iterate through and produce [1,3],[1,4],[2,3],[2,4].
+ *
+ */
+ static class SlotsIterator {
+ public final int pkPos;
+ private List<KeySlots> childSlots;
+ private List<SlotRangesIterator> slotRangesIterator;
+ private boolean firstCall = true;
+
+ SlotsIterator(List<KeySlots> childSlots, int pkPos) {
+ this.childSlots = childSlots;
+ this.pkPos = pkPos;
+ this.slotRangesIterator = Lists.newArrayListWithExpectedSize(childSlots.size() * 3 / 2);
+ for (int i = 0; i < childSlots.size(); i++) {
+ SlotRangesIterator iterator = new SlotRangesIterator(i);
+ slotRangesIterator.add(iterator);
+ iterator.initialize();
+ }
+ }
+
+ public KeySlot getSlot(int index) {
+ SlotRangesIterator slotRanges = slotRangesIterator.get(index);
+ return slotRanges.getSlot();
+ }
+
+ public KeyRange getRange(int index) {
+ SlotRangesIterator slotRanges = slotRangesIterator.get(index);
+ return slotRanges.getRange();
+ }
+
+ public boolean next() {
+ if (firstCall) {
+ boolean hasAny = false;
+ for (int i = 0; i < childSlots.size(); i++) {
+ hasAny |= this.slotRangesIterator.get(i).initialize();
+ }
+ firstCall = false;
+ return hasAny;
+ }
+ int i = 0;
+ while (i < childSlots.size() && !slotRangesIterator.get(i).next()) {
+ i++;
+ }
+ for (i = 0; i < childSlots.size(); i++) {
+ if (!this.slotRangesIterator.get(i).isWrapped()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private class SlotRangesIterator {
+ public int slotIndex;
+ public int rangeIndex;
+ public final KeySlots slots;
+ public boolean wrapped;
+
+ public SlotRangesIterator(int slotsIndex) {
+ this.slots = childSlots.get(slotsIndex);
+ }
+
+ public boolean isWrapped() {
+ return wrapped || !hasAny();
+ }
+
+ private boolean initialize() {
+ slotIndex = 0;
+ rangeIndex = 0;
+ while (slotIndex < slots.getSlots().size()
+ && (slots.getSlots().get(slotIndex) == null
+ || slots.getSlots().get(slotIndex).getKeyRanges().isEmpty()
+ || slots.getSlots().get(slotIndex).getPKPosition() != pkPos)) {
+ slotIndex++;
+ }
+ return hasAny();
+ }
+
+ private boolean hasAny() {
+ return slotIndex < slots.getSlots().size();
+ }
+
+ public KeySlot getSlot() {
+ if (!hasAny()) return null;
+ return slots.getSlots().get(slotIndex);
+ }
+
+ public KeyRange getRange() {
+ if (!hasAny()) return null;
+ return getSlot().getKeyRanges().get(rangeIndex);
+ }
+
+ public boolean next() {
+ if (!hasAny()) {
+ return false;
+ }
+ List<KeyRange> ranges = getSlot().getKeyRanges();
+ if ((rangeIndex = (rangeIndex + 1) % ranges.size()) == 0) {
+ do {
+ if (((slotIndex = (slotIndex + 1) % slots.getSlots().size()) == 0)) {
+ initialize();
+ wrapped = true;
return false;
}
- filled = true;
- }
+ } while (getSlot() == null || getSlot().getKeyRanges().isEmpty() || getSlot().getPKPosition() != pkPos);
}
+
+ return true;
}
}
- if (!filled) {
- slotArray[childPosition] = childSlot;
- }
- return true;
}
+ /**
+ * Ands together an arbitrary set of compiled expressions (represented as a list of KeySlots)
+ * by intersecting each unique combination among the childSlots.
+ * @param andExpression expressions being anded together
+ * @param childSlots compiled form of child expressions being anded together.
+ * @return
+ */
private KeySlots andKeySlots(AndExpression andExpression, List<KeySlots> childSlots) {
if(childSlots.isEmpty()) {
return null;
}
- int nColumns = table.getPKColumns().size();
- KeySlot[] keySlot = new KeySlot[nColumns];
- KeyRange minMaxRange = KeyRange.EVERYTHING_RANGE;
- List<Expression> minMaxExtractNodes = Lists.<Expression>newArrayList();
- int initPosition = (table.getBucketNum() ==null ? 0 : 1) + (this.context.getConnection().getTenantId() != null && table.isMultiTenant() ? 1 : 0) + (table.getViewIndexId() == null ? 0 : 1);
+ // Exit early if it's already been determined that one of the child slots cannot
+ // possibly be true.
boolean partialExtraction = andExpression.getChildren().size() != childSlots.size();
- for (KeySlots childSlot : childSlots) {
+ int nChildSlots = childSlots.size();
+ for (int i = 0; i < nChildSlots; i++) {
+ KeySlots childSlot = childSlots.get(i);
if (childSlot == EMPTY_KEY_SLOTS) {
return EMPTY_KEY_SLOTS;
}
+ // If any child slots represent partially extracted expressions, then carry
+ // that forward. An example of a partially extracted expression would be a
+ // RVC of (K1, K2, NonK3) in which only leading PK columns are extracted
+ // from the RVC.
partialExtraction |= childSlot.isPartialExtraction();
- // FIXME: get rid of this special-cased min/max range now that a key range can span multiple columns
- if (childSlot.getMinMaxRange() != null) { // Only set if in initial pk position
- // TODO: fix intersectSlots so that it works with RVCs. We'd just need to fill in the leading parts
- // of the key with the minMaxRange and then intersect the key parts that overlap.
- minMaxRange = minMaxRange.intersect(childSlot.getMinMaxRange());
- for (KeySlot slot : childSlot) {
+ }
+ boolean mayExtractNodes = true;
+ ImmutableBytesWritable ptr = context.getTempPtr();
+ RowKeySchema rowKeySchema = table.getRowKeySchema();
+ int nPkColumns = table.getPKColumns().size();
+ KeySlot[] keySlotArray = new KeySlot[nPkColumns];
+ int initPkPos = (table.getBucketNum() ==null ? 0 : 1) + (this.context.getConnection().getTenantId() != null && table.isMultiTenant() ? 1 : 0) + (table.getViewIndexId() == null ? 0 : 1);
+
+ List<List<List<KeyRange[]>>> slotsTrailingRanges = Lists.newArrayListWithExpectedSize(nPkColumns);
+ // Process all columns being ANDed in position order to guarantee
+ // we have all information for leading PK columns before we attempt
+ // to intersect them. For example:
+ // (A, B, C) >= (1, 2, 3) AND (B, C) < (4, 5) AND A = 1
+ // will processing slot 0 (i.e PK column A) across all children first,
+ // followed by slot 1 (PK column B), and finally slot 2 (C). This is
+ // done because we carry forward any constraints from preceding PK
+ // columns which may impact following PK columns. In the above example
+ // we'd carry forward that (B,C) >= (2,3) since we know that A is 1.
+ for (int pkPos = initPkPos; pkPos < nPkColumns; pkPos++) {
+ SlotsIterator iterator = new SlotsIterator(childSlots, pkPos);
+ OrderPreserving orderPreserving = null;
+ List<Expression> extractNodes = Lists.newArrayList();
+ List<KeyRange> keyRanges = Lists.newArrayList();
+ // This is the information carried forward as we process in PK order.
+ // It's parallel with the list of keyRanges.
+ List<KeyRange[]> trailingRangesList = Lists.<KeyRange[]>newArrayList();
+ KeyRange result = null;
+ TrailingRangeIterator trailingRangeIterator = new TrailingRangeIterator(initPkPos, pkPos, slotsTrailingRanges);
+ // Iterate through all combinations (i.e. constraints) for the PK slot being processed.
+ // For example, with (A = 1 OR A = 2) AND (A,B) > (1,2) AND C = 3, we'd process the
+ // following two combinations:
+ // A=1,(A,B) > (1,2)
+ // A=2,(A,B) > (1,2)
+ // If we have no constraints for a PK, then we still must iterate through the information
+ // that may have been rolled up based on the processing of previous PK slots. For example,
+ // in the above ANDed expressions, we have no constraint on B, but we would end up with
+ // rolled up information based on the B part of the (A,B) constraint.
+ while (iterator.next() || (trailingRangeIterator.hasNext() && result != KeyRange.EMPTY_RANGE)) {
+ result = null;
+ KeyRange[] trailingRanges = newTrailingRange();
+ for (int i = 0; i < nChildSlots && result != KeyRange.EMPTY_RANGE; i++) {
+ KeySlot slot = iterator.getSlot(i);
+ // Rollup the order preserving and concatenate the extracted expressions.
+ // Extracted expressions end up being removed from the AND expression at
+ // the top level call (pushKeyExpressionsToScan) with anything remaining
+ // ending up as a Filter (rather than contributing to the start/stop row
+ // of the scan.
if (slot != null) {
- // We can only definitely extract the expression nodes that start from the
- // leading PK column. They may get extracted at the end if we end up having
- // expressions matching the leading PK columns, but otherwise we'll be forced
- // to execute the expression in a filter.
- if (slot.getPKPosition() == initPosition) {
- minMaxExtractNodes.addAll(slot.getKeyPart().getExtractNodes());
- } else {
- if (!intersectSlots(keySlot, slot)) {
- return EMPTY_KEY_SLOTS;
+ KeyRange otherRange = iterator.getRange(i);
+ KeyRange range = result;
+ if (slot.getOrderPreserving() != null) {
+ orderPreserving = slot.getOrderPreserving().combine(orderPreserving);
+ }
+ if (slot.getKeyPart().getExtractNodes() != null) {
+ extractNodes.addAll(slot.getKeyPart().getExtractNodes());
+ }
+ // Keep a running intersection of the ranges we see. Note that the
+ // ranges are derived from constants appearing on the RHS of a comparison
+ // expression. For example, the expression A > 5 would produce a keyRange
+ // of (5, *) for slot 0 (assuming A is the leading PK column) If the result
+ // ends up as an empty key, that combination is ruled out. This is essentially
+ // doing constant reduction.
+ result = intersectRanges(pkPos, range, otherRange, trailingRanges);
+ }
+ }
+
+ if (result != KeyRange.EMPTY_RANGE) {
+ Map<KeyRange,List<KeyRange[]>> results = Maps.newHashMap();
+ trailingRangeIterator.init();
+ // Process all constraints that have been rolled up from previous
+ // processing of PK slots. This occurs for RVCs which span PK slots
+ // in which the leading part of the RVC is determined to be equal
+ // to a constant on the RHS.
+ while (trailingRangeIterator.hasNext()) {
+ // Loop through all combinations of values for all previously
+ // calculated slots.
+ do {
+ // Loop through all combinations of range constraints for the
+ // current combinations of values. If no valid combinations
+ // are found, we can rule out the result. We can also end up
+ // modifying the result if it has an intersection with the
+ // range constraints.
+ do {
+ KeyRange priorTrailingRange = trailingRangeIterator.getRange();
+ if (priorTrailingRange != KeyRange.EVERYTHING_RANGE) {
+ KeyRange[] intTrailingRanges = Arrays.copyOf(trailingRanges, trailingRanges.length);
+ // Intersect the current result with each range constraint. We essentially
+ // rule out the result when we find a constraint that has no intersection
+ KeyRange intResult = intersectRanges(pkPos, result, priorTrailingRange, intTrailingRanges);
+ if (intResult != KeyRange.EMPTY_RANGE) {
+ addResult(intResult, intTrailingRanges, results);
+ }
+ }
+ } while (trailingRangeIterator.nextTrailingRange());
+ } while (trailingRangeIterator.nextRange());
+ }
+ if (results.isEmpty() && result != null) { // No trailing range constraints
+ keyRanges.add(result);
+ trailingRangesList.add(trailingRanges);
+ } else {
+ mayExtractNodes &= results.size() <= 1;
+ for (Map.Entry<KeyRange,List<KeyRange[]>> entry : results.entrySet()) {
+ // Add same KeyRange with each KeyRange[] since the two lists are parallel
+ for (KeyRange[] trailingRange : entry.getValue()) {
+ keyRanges.add(entry.getKey());
+ trailingRangesList.add(trailingRange);
}
}
}
}
+ }
+
+ if (result == null && keyRanges.isEmpty()) {
+ slotsTrailingRanges.add(Collections.<List<KeyRange[]>>emptyList());
+ } else {
+ // If we encountered a result for this slot and
+ // there are no ranges, this is the degenerate case.
+ if (keyRanges.isEmpty()) {
+ return EMPTY_KEY_SLOTS;
+ }
+ // Similar to KeyRange.coalesce(), except we must combine together
+ // any rolled up constraints (as a list of KeyRanges) for a
+ // particular value (as they're coalesced together). We maintain
+ // these KeyRange constraints as a parallel list between keyRanges
+ // and trailingRangesList.
+ keyRanges = coalesceKeyRangesAndTrailingRanges(keyRanges, trailingRangesList, slotsTrailingRanges);
+ int maxSpan = 1;
+ for (KeyRange aRange : keyRanges) {
+ int span = rowKeySchema.computeMaxSpan(pkPos, aRange, context.getTempPtr());
+ if (span > maxSpan) {
+ maxSpan = span;
+ }
+ }
+ keySlotArray[pkPos] = new KeySlot(
+ new BaseKeyPart(table, table.getPKColumns().get(pkPos), mayExtractNodes ? extractNodes : Collections.<Expression>emptyList()),
+ pkPos,
+ maxSpan,
+ keyRanges,
+ orderPreserving);
+ }
+ }
+
+ // Filters trailing part of RVC based on ranges from PK columns after the one we're
+ // currently processing that may overlap with this range. For example, with a PK
+ // columns A,B,C and a range of A from [(1,2,3) - (4,5,6)] and B from (6-*), we
+ // can filter the trailing part of the RVC for A, because the trailing part of
+ // the RVC (2,3)-(5,6) does not intersect with (6-*). By removing the trailing
+ // part of the RVC, we end up with a range of A from [1-4] and B from (6-*) which
+ // enables us to use a skip scan.
+ for (int i = 0; i < keySlotArray.length; i++) {
+ KeySlot keySlot = keySlotArray[i];
+ if (keySlot == null) continue;
+ int pkSpan = keySlot.getPKSpan();
+ int pkPos = keySlot.getPKPosition();
+ boolean slotWasIntersected = false;
+ List<KeyRange> keyRanges = keySlot.getKeyRanges();
+ List<KeyRange> slotTrimmedResults = Lists.newArrayListWithExpectedSize(keyRanges.size());
+ for (KeyRange result : keyRanges) {
+ boolean resultWasIntersected = false;
+ Set<KeyRange> trimmedResults = Sets.newHashSetWithExpectedSize(keyRanges.size());
+ for (int trailingPkPos = pkPos+1; trailingPkPos < pkPos+pkSpan && trailingPkPos < nPkColumns; trailingPkPos++) {
+ KeySlot nextKeySlot = keySlotArray[trailingPkPos];
+ if (nextKeySlot == null) continue;
+ for (KeyRange trailingRange : nextKeySlot.getKeyRanges()) {
+ resultWasIntersected = true;
+ KeyRange intResult = intersectTrailing(result, pkPos, trailingRange, trailingPkPos);
+ if (intResult != KeyRange.EMPTY_RANGE) {
+ trimmedResults.add(intResult);
+ }
+ }
+ }
+ if (resultWasIntersected) {
+ slotWasIntersected = true;
+ slotTrimmedResults.addAll(trimmedResults);
+ mayExtractNodes &= trimmedResults.size() <= 1;
+ } else {
+ slotTrimmedResults.add(result);
+ }
+ }
+ if (slotTrimmedResults.isEmpty()) {
+ return EMPTY_KEY_SLOTS;
+ }
+ if (slotWasIntersected) {
+ // Re-coalesce the ranges and recalc the max span since the ranges may have changed
+ slotTrimmedResults = KeyRange.coalesce(slotTrimmedResults);
+ pkSpan = 1;
+ for (KeyRange trimmedResult : slotTrimmedResults) {
+ pkSpan = Math.max(pkSpan, rowKeySchema.computeMaxSpan(pkPos, trimmedResult, ptr));
+ }
+ }
+ List<Expression> extractNodes = mayExtractNodes ?
+ keySlotArray[pkPos].getKeyPart().getExtractNodes() : Collections.<Expression>emptyList();
+ keySlotArray[pkPos] = new KeySlot(
+ new BaseKeyPart(table, table.getPKColumns().get(pkPos), extractNodes),
+ pkPos,
+ pkSpan,
+ slotTrimmedResults,
+ keySlotArray[pkPos].getOrderPreserving());
+ }
+ List<KeySlot> keySlots = Arrays.asList(keySlotArray);
+ // If we have a salt column, skip that slot because
+ // they'll never be an expression that uses it directly.
+ keySlots = keySlots.subList(initPkPos, keySlots.size());
+ return new MultiKeySlot(keySlots, partialExtraction);
+ }
+
+ private KeyRange[] newTrailingRange() {
+ KeyRange[] trailingRanges = new KeyRange[table.getPKColumns().size()];
+ for (int i = 0; i < trailingRanges.length; i++) {
+ trailingRanges[i] = KeyRange.EVERYTHING_RANGE;
+ }
+ return trailingRanges;
+ }
+
+ private static void addResult(KeyRange result, KeyRange[] trailingRange, Map<KeyRange,List<KeyRange[]>> results) {
+ List<KeyRange[]> trailingRanges = Lists.<KeyRange[]>newArrayList(trailingRange);
+ List<KeyRange[]> priorTrailingRanges = results.put(result, trailingRanges);
+ if (priorTrailingRanges != null) {
+ // This is tricky case. We may have multiple possible values based on the rolled up range
+ // constraints from previous slots. We track unique ranges and concatenate together the
+ // trailing range data. If there's more than one element in the set (i.e. more than one
+ // possible result), we'll end up have more combinations than there actually are because
+ // the constraint only apply for a single value, not for *all* combinations (which is a
+ // limitation of our representation derived from what can be handled by our SkipScanFilter).
+ // For example, if we we've gathered these ranges so far in a three PK table: (1,2), (A,B)
+ // and have X as a constraint for value A and Y as a constraint for value B, we have the
+ // following possible combinations: 1AX, 2AX, 1BY, 2BY. However, our SkipScanFilter only
+ // supports identifying combinations for *all* combinations of (1,2),(A,B),(X,Y) or
+ // AX, 1AY, 1BX, 1BY, 2AX, 2AY, 2BX, 2BY. See WhereOptimizerTest.testNotRepresentableBySkipScan()
+ // for an example.
+ trailingRanges.addAll(priorTrailingRanges);
+ }
+ }
+
+ private List<KeyRange> coalesceKeyRangesAndTrailingRanges(List<KeyRange> keyRanges,
+ List<KeyRange[]> trailingRangesList, List<List<List<KeyRange[]>>> slotsTrailingRanges) {
+ List<Pair<KeyRange,List<KeyRange[]>>> pairs = coalesce(keyRanges, trailingRangesList);
+ List<List<KeyRange[]>> trailingRanges = Lists.newArrayListWithExpectedSize(pairs.size());
+ List<KeyRange>coalescedKeyRanges = Lists.newArrayListWithExpectedSize(pairs.size());
+ for (Pair<KeyRange,List<KeyRange[]>> pair : pairs) {
+ coalescedKeyRanges.add(pair.getFirst());
+ trailingRanges.add(pair.getSecond());
+ }
+ slotsTrailingRanges.add(trailingRanges);
+ return coalescedKeyRanges;
+ }
+
+ public static final Comparator<Pair<KeyRange,List<KeyRange[]>>> KEY_RANGE_PAIR_COMPARATOR = new Comparator<Pair<KeyRange,List<KeyRange[]>>>() {
+ @Override public int compare(Pair<KeyRange,List<KeyRange[]>> o1, Pair<KeyRange,List<KeyRange[]>> o2) {
+ return KeyRange.COMPARATOR.compare(o1.getFirst(), o2.getFirst());
+ }
+ };
+
+ private static boolean isEverythingRanges(KeyRange[] ranges) {
+ for (KeyRange range : ranges) {
+ if (range != KeyRange.EVERYTHING_RANGE) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static List<KeyRange[]> concat(List<KeyRange[]> list1, List<KeyRange[]> list2) {
+ if (list1.size() == 1 && isEverythingRanges(list1.get(0))) {
+ if (list2.size() == 1 && isEverythingRanges(list1.get(0))) {
+ return Collections.emptyList();
+ }
+ return list2;
+ }
+ if (list2.size() == 1 && isEverythingRanges(list2.get(0))) {
+ return list1;
+ }
+
+ List<KeyRange[]> newList = Lists.<KeyRange[]>newArrayListWithExpectedSize(list1.size()+list2.size());
+ newList.addAll(list1);
+ newList.addAll(list2);
+ return newList;
+ }
+
+ /**
+ * Similar to KeyRange.coelesce, but con
+ */
+ @NonNull
+ public static List<Pair<KeyRange,List<KeyRange[]>>> coalesce(List<KeyRange> keyRanges, List<KeyRange[]> trailingRangesList) {
+ List<Pair<KeyRange,List<KeyRange[]>>> tmp = Lists.newArrayListWithExpectedSize(keyRanges.size());
+ int nKeyRanges = keyRanges.size();
+ for (int i = 0; i < nKeyRanges; i++) {
+ KeyRange keyRange = keyRanges.get(i);
+ KeyRange[] trailingRange = trailingRangesList.get(i);
+ Pair<KeyRange,List<KeyRange[]>> pair = new Pair<KeyRange,List<KeyRange[]>>(keyRange,Lists.<KeyRange[]>newArrayList(trailingRange));
+ tmp.add(pair);
+ }
+ Collections.sort(tmp, KEY_RANGE_PAIR_COMPARATOR);
+ List<Pair<KeyRange,List<KeyRange[]>>> tmp2 = Lists.<Pair<KeyRange,List<KeyRange[]>>>newArrayListWithExpectedSize(tmp.size());
+ Pair<KeyRange,List<KeyRange[]>> range = tmp.get(0);
+ for (int i=1; i<tmp.size(); i++) {
+ Pair<KeyRange,List<KeyRange[]>> otherRange = tmp.get(i);
+ KeyRange intersect = range.getFirst().intersect(otherRange.getFirst());
+ if (KeyRange.EMPTY_RANGE == intersect) {
+ tmp2.add(range);
+ range = otherRange;
+ } else {
+ KeyRange newRange = range.getFirst().union(otherRange.getFirst());
+ range = new Pair<KeyRange,List<KeyRange[]>>(newRange,concat(range.getSecond(),otherRange.getSecond()));
+ }
+ }
+ tmp2.add(range);
+ List<Pair<KeyRange,List<KeyRange[]>>> tmp3 = Lists.<Pair<KeyRange,List<KeyRange[]>>>newArrayListWithExpectedSize(tmp2.size());
+ range = tmp2.get(0);
+ for (int i=1; i<tmp2.size(); i++) {
+ Pair<KeyRange,List<KeyRange[]>> otherRange = tmp2.get(i);
+ assert !range.getFirst().upperUnbound();
+ assert !otherRange.getFirst().lowerUnbound();
+ if (range.getFirst().isUpperInclusive() != otherRange.getFirst().isLowerInclusive()
+ && Bytes.equals(range.getFirst().getUpperRange(), otherRange.getFirst().getLowerRange())) {
+ KeyRange newRange = KeyRange.getKeyRange(
+ range.getFirst().getLowerRange(), range.getFirst().isLowerInclusive(),
+ otherRange.getFirst().getUpperRange(), otherRange.getFirst().isUpperInclusive());
+ range = new Pair<KeyRange,List<KeyRange[]>>(newRange,concat(range.getSecond(),otherRange.getSecond()));
} else {
- for (KeySlot slot : childSlot) {
- // The slot will be null if we have no condition for this slot
- if (slot != null && !intersectSlots(keySlot, slot)) {
- return EMPTY_KEY_SLOTS;
+ tmp3.add(range);
+ range = otherRange;
+ }
+ }
+ tmp3.add(range);
+
+ return tmp3;
+ }
+
+ /**
+ *
+ * Iterates over all unique combinations of the List<KeyRange[]> representing
+ * the constraints from previous slot positions. For example, if we have
+ * a RVC of (A,B) = (2,1), then if A=2, we know that B must be 1.
+ *
+ */
+ static class TrailingRangeIterator {
+ private final List<List<List<KeyRange[]>>> slotTrailingRangesList;
+ private final int[] rangePos;
+ private final int[] trailingRangePos;
+ private final int initPkPos;
+ private final int pkPos;
+ private int trailingRangePosIndex;
+ private int rangePosIndex;
+ private boolean hasMore = true;
+
+ TrailingRangeIterator (int initPkPos, int pkPos, List<List<List<KeyRange[]>>> slotsTrailingRangesList) {
+ this.slotTrailingRangesList = slotsTrailingRangesList;
+ int nSlots = pkPos - initPkPos;
+ rangePos = new int[nSlots];
+ trailingRangePos = new int[nSlots];
+ this.initPkPos = initPkPos;
+ this.pkPos = pkPos;
+ init();
+ }
+
+ public void init() {
+ Arrays.fill(rangePos, 0);
+ Arrays.fill(trailingRangePos, 0);
+ rangePosIndex = rangePos.length - 1;
+ trailingRangePosIndex = trailingRangePos.length - 1;
+ this.hasMore = pkPos > initPkPos && skipEmpty();
+ }
+
+ public boolean hasNext() {
+ return hasMore && skipEmpty();
+ }
+
+ public KeyRange getRange() {
+ if (!hasMore) {
+ throw new NoSuchElementException();
+ }
+ KeyRange priorTrailingRange = KeyRange.EVERYTHING_RANGE;
+ for (int priorPkPos = initPkPos; priorPkPos < pkPos; priorPkPos++) {
+ List<List<KeyRange[]>>trailingKeyRangesList = slotTrailingRangesList.get(priorPkPos-initPkPos);
+ if (!trailingKeyRangesList.isEmpty()) {
+ List<KeyRange[]> slotTrailingRanges = trailingKeyRangesList.get(rangePos[priorPkPos-initPkPos]);
+ if (!slotTrailingRanges.isEmpty()) {
+ KeyRange[] slotTrailingRange = slotTrailingRanges.get(trailingRangePos[priorPkPos-initPkPos]);
+ priorTrailingRange = priorTrailingRange.intersect(slotTrailingRange[pkPos]);
}
}
}
+
+ return priorTrailingRange;
+ }
+
+ private boolean skipEmptyTrailingRanges() {
+ while (trailingRangePosIndex >= 0 &&
+ (slotTrailingRangesList.get(trailingRangePosIndex).isEmpty()
+ || slotTrailingRangesList.get(trailingRangePosIndex).get(rangePos[trailingRangePosIndex]).isEmpty())) {
+ trailingRangePosIndex--;
+ }
+ if (trailingRangePosIndex >= 0) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean skipEmptyRanges() {
+ trailingRangePosIndex = trailingRangePos.length - 1;
+ while (rangePosIndex >= 0 &&
+ (slotTrailingRangesList.get(rangePosIndex).isEmpty())) {
+ rangePosIndex--;
+ }
+ return rangePosIndex >= 0;
+ }
+
+ private boolean skipEmpty() {
+ if (!hasMore || slotTrailingRangesList.isEmpty() || rangePosIndex < 0) {
+ return hasMore=false;
+ }
+ do {
+ if (skipEmptyTrailingRanges()) {
+ return true;
+ }
+ } while (skipEmptyRanges());
+ return hasMore = rangePosIndex >= 0;
+ }
+
+ public boolean nextRange() {
+ trailingRangePosIndex = trailingRangePos.length - 1;
+ while (rangePosIndex >= 0 &&
+ (slotTrailingRangesList.get(rangePosIndex).isEmpty()
+ || (rangePos[rangePosIndex] = (rangePos[rangePosIndex] + 1)
+ % slotTrailingRangesList.get(rangePosIndex).size()) == 0)) {
+ rangePosIndex--;
+ }
+ return rangePosIndex >= 0;
}
- if (!minMaxExtractNodes.isEmpty()) {
- if (keySlot[initPosition] == null) {
- keySlot[initPosition] = new KeySlot(new BaseKeyPart(table, table.getPKColumns().get(initPosition), minMaxExtractNodes), initPosition, 1, EVERYTHING_RANGES, null);
+ public boolean nextTrailingRange() {
+ while (trailingRangePosIndex >= 0 &&
+ (slotTrailingRangesList.get(trailingRangePosIndex).isEmpty()
+ || slotTrailingRangesList.get(trailingRangePosIndex).get(rangePos[trailingRangePosIndex]).isEmpty()
+ || (trailingRangePos[trailingRangePosIndex] = (trailingRangePos[trailingRangePosIndex] + 1)
+ % slotTrailingRangesList.get(trailingRangePosIndex).get(rangePos[trailingRangePosIndex]).size()) == 0)) {
+ trailingRangePosIndex--;
+ }
+ if (trailingRangePosIndex >= 0) {
+ return true;
+ }
+ return false;
+ }
+ }
+
+ private KeyRange intersectRanges(int pkPos, KeyRange range, KeyRange otherRange, KeyRange[] trailingRanges) {
+ // We need to initialize result to the other range rather than
+ // initializing it to EVERYTHING_RANGE to handle the IS NULL case.
+ // Otherwise EVERYTHING_RANGE intersected below with NULL_RANGE
+ // becomes an EMPTY_RANGE.
+ if (range == null) {
+ range = otherRange;
+ }
+ KeyRange result = range;
+ ImmutableBytesWritable ptr = context.getTempPtr();
+ RowKeySchema rowKeySchema = table.getRowKeySchema();
+ int minSpan = rowKeySchema.computeMinSpan(pkPos, result, ptr);
+ int otherMinSpan = rowKeySchema.computeMinSpan(pkPos, otherRange, ptr);
+ KeyRange otherClippedRange = otherRange;
+ KeyRange clippedRange = result;
+ if (minSpan != otherMinSpan && result != KeyRange.EVERYTHING_RANGE && otherRange != KeyRange.EVERYTHING_RANGE) {
+ if (otherMinSpan > minSpan) {
+ otherClippedRange = rowKeySchema.clipLeft(pkPos, otherRange, minSpan, ptr);
+ } else if (minSpan > otherMinSpan) {
+ clippedRange = rowKeySchema.clipLeft(pkPos, result, otherMinSpan, ptr);
+ }
+ }
+
+ // intersect result with otherRange
+ result = clippedRange.intersect(otherClippedRange);
+ if (result == KeyRange.EMPTY_RANGE) {
+ return result;
+ }
+ if (minSpan != otherMinSpan) {
+ // If trailing ranges are of different spans, intersect them at the common
+ // span and add remaining part of range used to trailing ranges
+ // Without the special case for single key values, the trailing ranges
+ // code doesn't work correctly for WhereOptimizerTest.testMultiSlotTrailingIntersect()
+ if (result.isSingleKey() && !(range.isSingleKey() && otherRange.isSingleKey())) {
+ int trailingPkPos = pkPos + Math.min(minSpan, otherMinSpan);
+ KeyRange trailingRange = getTrailingRange(rowKeySchema, trailingPkPos, minSpan > otherMinSpan ? range : otherRange, result, ptr);
+ trailingRanges[trailingPkPos] = trailingRanges[trailingPkPos].intersect(trailingRange);
} else {
- keySlot[initPosition] = keySlot[initPosition].concatExtractNodes(minMaxExtractNodes);
+ // Add back clipped part of range
+ if (otherMinSpan > minSpan) {
+ result = concatSuffix(result, otherRange);
+ } else if (minSpan > otherMinSpan) {
+ result = concatSuffix(result, range);
+ }
}
}
- List<KeySlot> keySlots = Arrays.asList(keySlot);
- // If we have a salt column, skip that slot because
- // they'll never be an expression contained by it.
- keySlots = keySlots.subList(initPosition, keySlots.size());
- return new MultiKeySlot(keySlots, minMaxRange == KeyRange.EVERYTHING_RANGE ? null : minMaxRange, partialExtraction);
+ return result;
+ }
+
+ private static KeyRange concatSuffix(KeyRange result, KeyRange otherRange) {
+ byte[] lowerRange = result.getLowerRange();
+ byte[] clippedLowerRange = lowerRange;
+ byte[] fullLowerRange = otherRange.getLowerRange();
+ if (!result.lowerUnbound() && Bytes.startsWith(fullLowerRange, clippedLowerRange)) {
+ lowerRange = fullLowerRange;
+ }
+ byte[] upperRange = result.getUpperRange();
+ byte[] clippedUpperRange = upperRange;
+ byte[] fullUpperRange = otherRange.getUpperRange();
+ if (!result.lowerUnbound() && Bytes.startsWith(fullUpperRange, clippedUpperRange)) {
+ upperRange = fullUpperRange;
+ }
+ if (lowerRange == clippedLowerRange && upperRange == clippedUpperRange) {
+ return result;
+ }
+ return KeyRange.getKeyRange(lowerRange, result.isLowerInclusive(), upperRange, result.isUpperInclusive());
+ }
+
+ /**
+ * Intersects an RVC that starts at pkPos with an overlapping range that starts at otherPKPos.
+ * For example, ((A, B) - (J, K)) intersected with (F - *) would return ((A,F) - (J, K))
+ * ((A, B) - (J, K)) intersected with (M - P) would return (A-J) since both of the trailing
+ * part of the RVC, B and K, do not intersect with B and K.
+ * @param result an RVC expression starting from pkPos and with length of at least otherPKPos - pkPos.
+ * @param pkPos the PK position of the leading part of the RVC expression
+ * @param otherRange the other range to intersect with the overlapping part of the RVC.
+ * @param otherPKPos the PK position of the leading part of the other range
+ * @return resulting KeyRange from the intersection, potentially an empty range if the result RVC
+ * is a single key and the trailing part of the key does not intersect with the RVC.
+ */
+ private KeyRange intersectTrailing(KeyRange result, int pkPos, KeyRange otherRange, int otherPKPos) {
+ RowKeySchema rowKeySchema = table.getRowKeySchema();
+ ImmutableBytesWritable ptr = context.getTempPtr();
+ int separatorLength = table.getPKColumns().get(otherPKPos-1).getDataType().isFixedWidth() ? 0 : 1;
+ boolean lowerInclusive = result.isLowerInclusive();
+ byte[] lowerRange = result.getLowerRange();
+ ptr.set(lowerRange);
+ // Position ptr at the point at which the two ranges overlap
+ if (rowKeySchema.position(ptr, pkPos, otherPKPos)) {
+ int lowerOffset = ptr.getOffset();
+ // Increase the length of the ptr to include the entire trailing bytes
+ ptr.set(ptr.get(), lowerOffset, lowerRange.length - lowerOffset);
+ byte[] trailingBytes = ptr.copyBytes();
+
+ // Special case for single key since single keys of different span lengths
+ // will never overlap. We do not need to process both the lower and upper
+ // ranges since they are the same.
+ if (result.isSingleKey() && otherRange.isSingleKey()) {
+ // Find the span of the trailing bytes as it could be more than one.
+ // We need this to determine if the slot at the last position would
+ // have a separator byte (i.e. is variable length).
+ int pos = otherPKPos;
+ rowKeySchema.iterator(trailingBytes, ptr, otherPKPos);
+ while (rowKeySchema.next(ptr, pos, trailingBytes.length) != null) {
+ pos++;
+ }
+ byte[] otherLowerRange = otherRange.getLowerRange();
+ boolean isFixedWidthAtEnd = table.getPKColumns().get(pos).getDataType().isFixedWidth();
+ // If the otherRange starts with the overlapping trailing byte *and* we're comparing
+ // the entire key (i.e. not just a leading subset), then we have an intersection.
+ if (Bytes.startsWith(otherLowerRange, trailingBytes) &&
+ (isFixedWidthAtEnd ||
+ otherLowerRange.length == trailingBytes.length ||
+ otherLowerRange[trailingBytes.length] == QueryConstants.SEPARATOR_BYTE)) {
+ return result;
+ }
+ // Otherwise, there is no overlap
+ return KeyRange.EMPTY_RANGE;
+ }
+ // If we're not dealing with single keys, then we can use our normal intersection
+ if (otherRange.intersect(KeyRange.getKeyRange(trailingBytes)) == KeyRange.EMPTY_RANGE) {
+ // Exit early since the upper range is the same as the lower range
+ if (result.isSingleKey()) {
+ return KeyRange.EMPTY_RANGE;
+ }
+ ptr.set(result.getLowerRange(), 0, lowerOffset - separatorLength);
+ lowerRange = ptr.copyBytes();
+ }
+ }
+ boolean upperInclusive = result.isUpperInclusive();
+ byte[] upperRange = result.getUpperRange();
+ ptr.set(upperRange);
+ if (rowKeySchema.position(ptr, pkPos, otherPKPos)) {
+ int upperOffset = ptr.getOffset();
+ ptr.set(ptr.get(), upperOffset, upperRange.length - upperOffset);
+ if (otherRange.intersect(KeyRange.getKeyRange(ptr.copyBytes())) == KeyRange.EMPTY_RANGE) {
+ ptr.set(ptr.get(), 0, upperOffset - separatorLength);
+ upperRange = ptr.copyBytes();
+ }
+ }
+ if (lowerRange == result.getLowerRange() && upperRange == result.getUpperRange()) {
+ return result;
+ }
+ KeyRange range = KeyRange.getKeyRange(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ return range;
}
private KeySlots orKeySlots(OrExpression orExpression, List<KeySlots> childSlots) {
@@ -735,7 +1420,6 @@ public class WhereOptimizer {
// TODO: Have separate list for single span versus multi span
// For multi-span, we only need to keep a single range.
List<KeyRange> slotRanges = Lists.newArrayList();
- KeyRange minMaxRange = KeyRange.EMPTY_RANGE;
for (KeySlots childSlot : childSlots) {
if (childSlot == EMPTY_KEY_SLOTS) {
// TODO: can this ever happen and can we safely filter the expression tree?
@@ -745,88 +1429,45 @@ public class WhereOptimizer {
// if all sub-expressions have been completely extracted. Otherwise, we must
// leave the OR as a post filter.
partialExtraction |= childSlot.isPartialExtraction();
- if (childSlot.getMinMaxRange() != null) {
- if (!slotRanges.isEmpty() && thePosition != initialPos) { // ORing together rvc in initial slot with other slots
- return null;
+ // TODO: Do the same optimization that we do for IN if the childSlots specify a fully qualified row key
+ for (KeySlot slot : childSlot.getSlots()) {
+ if (slot == null) {
+ continue;
}
- minMaxRange = minMaxRange.union(childSlot.getMinMaxRange());
- thePosition = initialPos;
- for (KeySlot slot : childSlot) {
- if (slot != null) {
- slotExtractNodes.addAll(slot.getKeyPart().getExtractNodes());
- }
- }
- } else {
- // TODO: Do the same optimization that we do for IN if the childSlots specify a fully qualified row key
- for (KeySlot slot : childSlot) {
- if (slot == null) {
- continue;
- }
- /*
- * If we see a different PK column than before, we can't
- * optimize it because our SkipScanFilter only handles
- * top level expressions that are ANDed together (where in
- * the same column expressions may be ORed together).
- * For example, WHERE a=1 OR b=2 cannot be handled, while
- * WHERE (a=1 OR a=2) AND (b=2 OR b=3) can be handled.
- * TODO: We could potentially handle these cases through
- * multiple, nested SkipScanFilters, where each OR expression
- * is handled by its own SkipScanFilter and the outer one
- * increments the child ones and picks the one with the smallest
- * key.
- */
- if (thePosition == -1) {
- theSlot = slot;
- thePosition = slot.getPKPosition();
- } else if (thePosition != slot.getPKPosition()) {
- return null;
- }
- slotExtractNodes.addAll(slot.getKeyPart().getExtractNodes());
- slotRanges.addAll(slot.getKeyRanges());
+ /*
+ * If we see a different PK column than before, we can't
+ * optimize it because our SkipScanFilter only handles
+ * top level expressions that are ANDed together (where in
+ * the same column expressions may be ORed together).
+ * For example, WHERE a=1 OR b=2 cannot be handled, while
+ * WHERE (a=1 OR a=2) AND (b=2 OR b=3) can be handled.
+ * TODO: We could potentially handle these cases through
+ * multiple, nested SkipScanFilters, where each OR expression
+ * is handled by its own SkipScanFilter and the outer one
+ * increments the child ones and picks the one with the smallest
+ * key.
+ */
+ if (thePosition == -1) {
+ theSlot = slot;
+ thePosition = slot.getPKPosition();
+ } else if (thePosition != slot.getPKPosition()) {
+ return null;
}
+ slotExtractNodes.addAll(slot.getKeyPart().getExtractNodes());
+ slotRanges.addAll(slot.getKeyRanges());
}
}
if (thePosition == -1) {
return null;
}
- // With a mix of both, we can't use skip scan, so empty out the union
- // and only extract the min/max nodes.
- if (!slotRanges.isEmpty() && minMaxRange != KeyRange.EMPTY_RANGE) {
- boolean clearExtracts = false;
- // Union the minMaxRanges together with the slotRanges.
- for (KeyRange range : slotRanges) {
- if (!clearExtracts) {
- /*
- * Detect when to clear the extract nodes by determining if there
- * are gaps left by combining the ranges. If there are gaps, we
- * cannot extract the nodes, but must them as filters instead.
- */
- KeyRange intersection = minMaxRange.intersect(range);
- if (intersection == KeyRange.EMPTY_RANGE
- || !range.equals(intersection.union(range))
- || !minMaxRange.equals(intersection.union(minMaxRange))) {
- clearExtracts = true;
- }
- }
- minMaxRange = minMaxRange.union(range);
- }
- if (clearExtracts) {
- partialExtraction = true;
- slotExtractNodes = Collections.emptyList();
- }
- slotRanges = Collections.emptyList();
- }
if (theSlot == null) {
theSlot = new KeySlot(new BaseKeyPart(table, table.getPKColumns().get(initialPos), slotExtractNodes), initialPos, 1, EVERYTHING_RANGES, null);
- } else if (minMaxRange != KeyRange.EMPTY_RANGE && !slotExtractNodes.isEmpty()) {
- theSlot = theSlot.concatExtractNodes(slotExtractNodes);
}
return newKeyParts(
theSlot,
partialExtraction ? slotExtractNodes : Collections.<Expression>singletonList(orExpression),
- slotRanges.isEmpty() ? EVERYTHING_RANGES : KeyRange.coalesce(slotRanges),
- minMaxRange == KeyRange.EMPTY_RANGE ? null : minMaxRange);
+ slotRanges.isEmpty() ? EVERYTHING_RANGES : KeyRange.coalesce(slotRanges));
}
private final PTable table;
@@ -847,7 +1488,7 @@ public class WhereOptimizer {
if (childParts.isEmpty()) {
return null;
}
- return newCoerceKeyPart(childParts.get(0).iterator().next(), node);
+ return newCoerceKeyPart(childParts.get(0).getSlots().get(0), node);
}
@Override
@@ -910,10 +1551,11 @@ public class WhereOptimizer {
}
Expression rhs = node.getChildren().get(1);
KeySlots childSlots = childParts.get(0);
- KeySlot childSlot = childSlots.iterator().next();
+ KeySlot childSlot = childSlots.getSlots().get(0);
KeyPart childPart = childSlot.getKeyPart();
- SortOrder sortOrder = childPart.getColumn().getSortOrder();
- CompareOp op = sortOrder.transform(node.getFilterOp());
+ //SortOrder sortOrder = childPart.getColumn().getSortOrder();
+ CompareOp op = node.getFilterOp();
+ //CompareOp op = sortOrder.transform(node.getFilterOp());
KeyRange keyRange = childPart.getKeyRange(op, rhs);
return newKeyParts(childSlot, node, keyRange);
}
@@ -935,7 +1577,7 @@ public class WhereOptimizer {
if (childParts.isEmpty()) {
return null;
}
- return newScalarFunctionKeyPart(childParts.get(0).iterator().next(), node);
+ return newScalarFunctionKeyPart(childParts.get(0).getSlots().get(0), node);
}
@Override
@@ -957,7 +1599,7 @@ public class WhereOptimizer {
}
// for SUBSTR(<column>,1,3) LIKE 'foo%'
KeySlots childSlots = childParts.get(0);
- KeySlot childSlot = childSlots.iterator().next();
+ KeySlot childSlot = childSlots.getSlots().get(0);
final String startsWith = node.getLiteralPrefix();
SortOrder sortOrder = node.getChildren().get(0).getSortOrder();
byte[] key = PVarchar.INSTANCE.toBytes(startsWith, sortOrder);
@@ -990,9 +1632,12 @@ public class WhereOptimizer {
lowerRange = Arrays.copyOf(lowerRange, lowerRange.length+1);
lowerRange[lowerRange.length-1] = QueryConstants.SEPARATOR_BYTE;
}
- KeyRange keyRange = type.getKeyRange(lowerRange, true, upperRange, false);
+ KeyRange range = type.getKeyRange(lowerRange, true, upperRange, false);
+ if (column.getSortOrder() == SortOrder.DESC) {
+ range = range.invert();
+ }
// Only extract LIKE expression if pattern ends with a wildcard and everything else was extracted
- return newKeyParts(childSlot, node.endsWithOnlyWildcard() ? node : null, keyRange);
+ return newKeyParts(childSlot, node.endsWithOnlyWildcard() ? node : null, range);
}
@Override
@@ -1008,7 +1653,7 @@ public class WhereOptimizer {
List<Expression> keyExpressions = node.getKeyExpressions();
Set<KeyRange> ranges = Sets.newHashSetWithExpectedSize(keyExpressions.size());
- KeySlot childSlot = childParts.get(0).iterator().next();
+ KeySlot childSlot = childParts.get(0).getSlots().get(0);
KeyPart childPart = childSlot.getKeyPart();
// Handles cases like WHERE substr(foo,1,3) IN ('aaa','bbb')
for (Expression key : keyExpressions) {
@@ -1020,7 +1665,7 @@ public class WhereOptimizer {
ranges.add(range);
}
}
- return newKeyParts(childSlot, node, new ArrayList<KeyRange>(ranges), null);
+ return newKeyParts(childSlot, node, new ArrayList<KeyRange>(ranges));
}
@Override
@@ -1034,10 +1679,12 @@ public class WhereOptimizer {
return null;
}
KeySlots childSlots = childParts.get(0);
- KeySlot childSlot = childSlots.iterator().next();
+ KeySlot childSlot = childSlots.getSlots().get(0);
PColumn column = childSlot.getKeyPart().getColumn();
PDataType type = column.getDataType();
boolean isFixedWidth = type.isFixedWidth();
+ // Nothing changes for IS NULL and IS NOT NULL when DESC since
+ // we represent NULL the same way for ASC and DESC
if (isFixedWidth) { // if column can't be null
return node.isNegate() ? null :
newKeyParts(childSlot, node, type.getKeyRange(new byte[SchemaUtil.getFixedByteSize(column)], true,
@@ -1048,9 +1695,25 @@ public class WhereOptimizer {
}
}
- private static interface KeySlots extends Iterable<KeySlot> {
- @Override public Iterator<KeySlot> iterator();
- public KeyRange getMinMaxRange();
+ /**
+ *
+ * Top level data structure used to drive the formation
+ * of the start/stop row of scans, essentially taking the
+ * expression tree of a WHERE clause and producing the
+ * ScanRanges instance during query compilation.
+ *
+ */
+ public static interface KeySlots {
+
+ /**
+ * List of slots that store binding of constant values
+ * for primary key columns. For example:
+ * WHERE pk1 = 'foo' and pk2 = 'bar'
+ * would produce two KeySlot instances that store that
+ * pk1 = 'foo' and pk2 = 'bar'.
+ * @return
+ */
+ public List<KeySlot> getSlots();
/**
* Tracks whether or not the contained KeySlot(s) contain
* a slot that includes only a partial extraction of the
@@ -1063,19 +1726,25 @@ public class WhereOptimizer {
*/
public boolean isPartialExtraction();
}
-
- private final class KeySlot {
- private final int pkPosition;
- private final int pkSpan;
- private final KeyPart keyPart;
+
+ /**
+ *
+ * Used during query compilation to represent the constant value of a
+ * primary key column based on expressions in the WHERE clause. These
+ * are combined together during the compilation of ANDs and ORs to
+ * to produce the start and stop scan range.
+ *
+ */
+ static final class KeySlot {
+ private final int pkPosition; // Position in primary key
+ private final int pkSpan; // Will be > 1 for RVC
+ private final KeyPart keyPart; // Used to produce the KeyRanges below
+ // Multiple ranges means values that have been ORed together
private final List<KeyRange> keyRanges;
- private final OrderPreserving orderPreserving;
-
- private KeySlot(KeyPart keyPart, int pkPosition, int pkSpan, List<KeyRange> keyRanges) {
- this (keyPart, pkPosition, pkSpan, keyRanges, OrderPreserving.YES);
- }
+ // If order rows returned from scan will match desired order declared in query
+ private final OrderPreserving orderPreserving;
- private KeySlot(KeyPart keyPart, int pkPosition, int pkSpan, List<KeyRange> keyRanges, OrderPreserving orderPreserving) {
+ KeySlot(KeyPart keyPart, int pkPosition, int pkSpan, List<KeyRange> keyRanges, OrderPreserving orderPreserving) {
this.pkPosition = pkPosition;
this.pkSpan = pkSpan;
this.keyPart = keyPart;
@@ -1109,100 +1778,29 @@ public class WhereOptimizer {
this.getOrderPreserving());
}
- public final KeySlot intersect(KeySlot that) {
- if (this.getPKSpan() == 1 && that.getPKSpan() == 1) {
- if (this.getPKPosition() != that.getPKPosition()) {
- throw new IllegalArgumentException("Position must be equal for intersect");
- }
- List<KeyRange> keyRanges = KeyRange.intersect(this.getKeyRanges(), that.getKeyRanges());
- if (isDegenerate(keyRanges)) {
- return null;
- }
- return new KeySlot(
- new BaseKeyPart(this.getKeyPart().getTable(), this.getKeyPart().getColumn(),
- SchemaUtil.concat(this.getKeyPart().getExtractNodes(),
- that.getKeyPart().getExtractNodes())),
- this.getPKPosition(),
- this.getPKSpan(),
- keyRanges,
- this.getOrderPreserving());
- } else {
- // Assumes that only single keys occur in RVCs (i.e. when a PK spans columns)
- assert(this.getPKSpan() > 1);
- assert(this.getPKPosition() <= that.getPKPosition());
- ImmutableBytesWritable ptr = context.getTempPtr();
- RowKeySchema schema = table.getRowKeySchema();
- if (this.getPKSpan() > 1 && that.getPKSpan() > 1) {
- // TODO: Trickiest case: both key slots are multi-span RVCs.
- // Punt for now: we could intersect these, but it'd be a fair amount of code.
- // Instead, just keep the original slot and don't extract
- // the other expressions (so they'll be evaluated row by row).
- return this;
- } else {
- assert(this.getPKSpan() > 1);
- assert(this.getPKPosition() <= that.getPKPosition());
- List<KeyRange> newKeyRanges = Lists.newArrayListWithExpectedSize(this.getKeyRanges().size());
- // We know we have a set of RVCs (i.e. multi-span key ranges)
- // Get the PK slot value in the RVC for the position of the other KeySlot
- // If they don't intersect, we cannot have a match for the RVC, so filter it out.
- // Otherwise, we keep it.
- for (KeyRange keyRange : this.getKeyRanges()) {
- if (keyRange == KeyRange.EVERYTHING_RANGE) {
- return this;
- }
- assert(keyRange.isSingleKey());
- byte[] key = keyRange.getLowerRange();
- int position = this.getPKPosition();
- int thatPosition = that.getPKPosition();
- ptr.set(key);
- if (schema.position(ptr, position, thatPosition)) {
- // Create a range just for the overlapping column
- List<KeyRange> slotKeyRanges = Collections.singletonList(KeyRange.getKeyRange(ByteUtil.copyKeyBytesIfNecessary(ptr)));
- // Intersect with other ranges and add to list if it overlaps
- if (!isDegenerate(KeyRange.intersect(slotKeyRanges, that.getKeyRanges()))) {
- newKeyRanges.add(keyRange);
- }
- }
- }
- if (isDegenerate(newKeyRanges)) {
- return null;
- }
- return new KeySlot(
- new BaseKeyPart(this.getKeyPart().getTable(), this.getKeyPart().getColumn(),
- SchemaUtil.concat(this.getKeyPart().getExtractNodes(),
- that.getKeyPart().getExtractNodes())),
- this.getPKPosition(),
- this.getPKSpan(),
- newKeyRanges,
- this.getOrderPreserving());
- }
- }
- }
-
public OrderPreserving getOrderPreserving() {
return orderPreserving;
}
}
- private static class MultiKeySlot implements KeySlots {
+ /**
+ *
+ * Implementation of KeySlots for AND and OR expressions. The
+ * List<KeySlot> will be in PK order.
+ *
+ */
+ public static class MultiKeySlot implements KeySlots {
private final List<KeySlot> childSlots;
- private final KeyRange minMaxRange;
private final boolean partialExtraction;
- private MultiKeySlot(List<KeySlot> childSlots, KeyRange minMaxRange, boolean partialExtraction) {
+ private MultiKeySlot(List<KeySlot> childSlots, boolean partialExtraction) {
this.childSlots = childSlots;
- this.minMaxRange = minMaxRange;
this.partialExtraction = partialExtraction;
}
@Override
- public Iterator<KeySlot> iterator() {
- return childSlots.iterator();
- }
-
- @Override
- public KeyRange getMinMaxRange() {
- return minMaxRange;
+ public List<KeySlot> getSlots() {
+ return childSlots;
}
@Override
@@ -1211,11 +1809,15 @@ public class WhereOptimizer {
}
}
- private class SingleKeySlot implements KeySlots {
- private final KeySlot slot;
- private final KeyRange minMaxRange;
+ /**
+ *
+ * Implementation of KeySlots for a constant value,
+ *
+ */
+ public static class SingleKeySlot implements KeySlots {
+ private final List<KeySlot> slots;
- private SingleKeySlot(KeyPart part, int pkPosition, List<KeyRange> ranges) {
+ SingleKeySlot(KeyPart part, int pkPosition, List<KeyRange> ranges) {
this(part, pkPosition, 1, ranges);
}
@@ -1224,36 +1826,26 @@ public class WhereOptimizer {
}
private SingleKeySlot(KeyPart part, int pkPosition, int pkSpan, List<KeyRange> ranges) {
- this(part,pkPosition,pkSpan,ranges, null, null);
+ this(part,pkPosition,pkSpan,ranges, null);
}
private SingleKeySlot(KeyPart part, int pkPosition, int pkSpan, List<KeyRange> ranges, OrderPreserving orderPreserving) {
- this(part,pkPosition,pkSpan,ranges, null, orderPreserving);
- }
-
- private SingleKeySlot(KeyPart part, int pkPosition, int pkSpan, List<KeyRange> ranges, KeyRange minMaxRange, OrderPreserving orderPreserving) {
- this.slot = new KeySlot(part, pkPosition, pkSpan, ranges, orderPreserving);
- this.minMaxRange = minMaxRange;
+ this.slots = Collections.singletonList(new KeySlot(part, pkPosition, pkSpan, ranges, orderPreserving));
}
@Override
- public Iterator<KeySlot> iterator() {
- return Iterators.<KeySlot>singletonIterator(slot);
- }
-
- @Override
- public KeyRange getMinMaxRange() {
- return minMaxRange;
+ public List<KeySlot> getSlots() {
+ return slots;
}
@Override
public boolean isPartialExtraction() {
- return this.slot.getKeyPart().getExtractNodes().isEmpty();
+ return this.slots.get(0).getKeyPart().getExtractNodes().isEmpty();
}
}
- private static class BaseKeyPart implements KeyPart {
+ public static class BaseKeyPart implements KeyPart {
@Override
public KeyRange getKeyRange(CompareOp op, Expression rhs) {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
@@ -1264,11 +1856,17 @@ public class WhereOptimizer {
Integer length = getColumn().getMaxLength();
<TRUNCATED>
[3/4] phoenix git commit: PHOENIX-3383 Comparison between descending
row keys used in RVC is reverse
Posted by ja...@apache.org.
PHOENIX-3383 Comparison between descending row keys used in RVC is reverse
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/aee568be
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/aee568be
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/aee568be
Branch: refs/heads/master
Commit: aee568beb02cdf983bb10889902c338ea016e6c9
Parents: d35a751
Author: James Taylor <ja...@apache.org>
Authored: Thu Jul 5 21:38:28 2018 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Wed Jul 11 07:28:47 2018 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/QueryMoreIT.java | 12 +-
.../org/apache/phoenix/compile/KeyPart.java | 2 -
.../org/apache/phoenix/compile/ScanRanges.java | 77 +-
.../apache/phoenix/compile/WhereOptimizer.java | 1304 +++++++++++++-----
.../PhoenixTxIndexMutationGenerator.java | 2 +-
.../expression/function/FunctionExpression.java | 10 +-
.../expression/function/InvertFunction.java | 19 +-
.../expression/function/PrefixFunction.java | 6 +-
.../expression/function/RTrimFunction.java | 6 +-
.../function/RoundDateExpression.java | 22 +-
.../function/RoundDecimalExpression.java | 7 +-
.../phoenix/iterate/BaseResultIterators.java | 4 +-
.../apache/phoenix/iterate/ExplainTable.java | 10 -
.../java/org/apache/phoenix/query/KeyRange.java | 28 +-
.../org/apache/phoenix/schema/RowKeySchema.java | 78 ++
.../phoenix/compile/QueryCompilerTest.java | 2 +-
.../phoenix/compile/QueryOptimizerTest.java | 5 +-
.../TenantSpecificViewIndexCompileTest.java | 8 +-
.../phoenix/compile/WhereOptimizerTest.java | 359 ++++-
.../RoundFloorCeilExpressionsTest.java | 59 +-
.../apache/phoenix/query/KeyRangeClipTest.java | 2 +-
.../org/apache/phoenix/query/QueryPlanTest.java | 8 +-
.../apache/phoenix/schema/RowKeySchemaTest.java | 48 +
23 files changed, 1567 insertions(+), 511 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
index 9109c12..04272fa 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryMoreIT.java
@@ -372,9 +372,6 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
}
}
- // FIXME: this repros PHOENIX-3382, but turned up two more issues:
- // 1) PHOENIX-3383 Comparison between descending row keys used in RVC is reverse
- // 2) PHOENIX-3384 Optimize RVC expressions for non leading row key columns
@Test
public void testRVCOnDescWithLeadingPKEquality() throws Exception {
final Connection conn = DriverManager.getConnection(getUrl());
@@ -398,14 +395,11 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES ('org1',1,'02')");
conn.commit();
- // FIXME: PHOENIX-3383
- // This comparison is really backwards: it should be (score, entity_id) < (2, '04'),
- // but because we're matching a descending key, our comparison has to be switched.
try (Statement stmt = conn.createStatement()) {
final ResultSet rs = stmt.executeQuery("SELECT entity_id, score\n" +
"FROM " + fullTableName + "\n" +
"WHERE organization_id = 'org1'\n" +
- "AND (score, entity_id) > (2, '04')\n" +
+ "AND (score, entity_id) < (2, '04')\n" +
"ORDER BY score DESC, entity_id DESC\n" +
"LIMIT 3");
assertTrue(rs.next());
@@ -416,13 +410,11 @@ public class QueryMoreIT extends ParallelStatsDisabledIT {
assertEquals(1.0, rs.getDouble(2), 0.001);
assertFalse(rs.next());
}
- // FIXME: PHOENIX-3384
- // It should not be necessary to specify organization_id in this query
try (Statement stmt = conn.createStatement()) {
final ResultSet rs = stmt.executeQuery("SELECT entity_id, score\n" +
"FROM " + fullTableName + "\n" +
"WHERE organization_id = 'org1'\n" +
- "AND (organization_id, score, entity_id) > ('org1', 2, '04')\n" +
+ "AND (organization_id, score, entity_id) < ('org1', 2, '04')\n" +
"ORDER BY score DESC, entity_id DESC\n" +
"LIMIT 3");
assertTrue(rs.next());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
index 55385f2..6cf9938 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/KeyPart.java
@@ -33,8 +33,6 @@ import org.apache.phoenix.schema.PTable;
* between a built-in function and the setting of the scan key
* during query compilation.
*
- *
- * @since 0.12
*/
public interface KeyPart {
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/aee568be/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
index 019f15d..1732fce 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.filter.SkipScanFilter;
@@ -45,7 +44,6 @@ import org.apache.phoenix.schema.types.PDataType.PDataCodec;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.ScanUtil;
-import org.apache.phoenix.util.ScanUtil.BytesComparator;
import org.apache.phoenix.util.SchemaUtil;
import com.google.common.base.Throwables;
@@ -56,30 +54,30 @@ import com.google.common.collect.Lists;
public class ScanRanges {
private static final List<List<KeyRange>> EVERYTHING_RANGES = Collections.<List<KeyRange>>emptyList();
private static final List<List<KeyRange>> NOTHING_RANGES = Collections.<List<KeyRange>>singletonList(Collections.<KeyRange>singletonList(KeyRange.EMPTY_RANGE));
- public static final ScanRanges EVERYTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, KeyRange.EVERYTHING_RANGE, KeyRange.EVERYTHING_RANGE, false, false, null, null);
- public static final ScanRanges NOTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, KeyRange.EMPTY_RANGE, KeyRange.EMPTY_RANGE, false, false, null, null);
+ public static final ScanRanges EVERYTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,EVERYTHING_RANGES, KeyRange.EVERYTHING_RANGE, false, false, null, null);
+ public static final ScanRanges NOTHING = new ScanRanges(null,ScanUtil.SINGLE_COLUMN_SLOT_SPAN,NOTHING_RANGES, KeyRange.EMPTY_RANGE, false, false, null, null);
private static final Scan HAS_INTERSECTION = new Scan();
public static ScanRanges createPointLookup(List<KeyRange> keys) {
- return ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1);
+ return ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, null, true, -1);
}
// For testing
public static ScanRanges createSingleSpan(RowKeySchema schema, List<List<KeyRange>> ranges) {
- return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), KeyRange.EVERYTHING_RANGE, null, true, -1);
+ return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), null, true, -1);
}
// For testing
public static ScanRanges createSingleSpan(RowKeySchema schema, List<List<KeyRange>> ranges, Integer nBuckets, boolean useSkipSan) {
- return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), KeyRange.EVERYTHING_RANGE, nBuckets, useSkipSan, -1);
+ return create(schema, ranges, ScanUtil.getDefaultSlotSpans(ranges.size()), nBuckets, useSkipSan, -1);
}
- public static ScanRanges create(RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan, KeyRange minMaxRange, Integer nBuckets, boolean useSkipScan, int rowTimestampColIndex) {
+ public static ScanRanges create(RowKeySchema schema, List<List<KeyRange>> ranges, int[] slotSpan, Integer nBuckets, boolean useSkipScan, int rowTimestampColIndex) {
int offset = nBuckets == null ? 0 : SaltingUtil.NUM_SALTING_BYTES;
int nSlots = ranges.size();
- if (nSlots == offset && minMaxRange == KeyRange.EVERYTHING_RANGE) {
+ if (nSlots == offset) {
return EVERYTHING;
- } else if (minMaxRange == KeyRange.EMPTY_RANGE || (nSlots == 1 + offset && ranges.get(offset).size() == 1 && ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
+ } else if ((nSlots == 1 + offset && ranges.get(offset).size() == 1 && ranges.get(offset).get(0) == KeyRange.EMPTY_RANGE)) {
return NOTHING;
}
TimeRange rowTimestampRange = getRowTimestampColumnRange(ranges, schema, rowTimestampColIndex);
@@ -88,22 +86,9 @@ public class ScanRanges {
// TODO: consider keeping original to use for serialization as it would be smaller?
List<byte[]> keys = ScanRanges.getPointKeys(ranges, slotSpan, schema, nBuckets);
List<KeyRange> keyRanges = Lists.newArrayListWithExpectedSize(keys.size());
- KeyRange unsaltedMinMaxRange = minMaxRange;
- if (nBuckets != null && minMaxRange != KeyRange.EVERYTHING_RANGE) {
- unsaltedMinMaxRange = KeyRange.getKeyRange(
- stripPrefix(minMaxRange.getLowerRange(),offset),
- minMaxRange.lowerUnbound(),
- stripPrefix(minMaxRange.getUpperRange(),offset),
- minMaxRange.upperUnbound());
- }
// We have full keys here, so use field from our varbinary schema
- BytesComparator comparator = ScanUtil.getComparator(SchemaUtil.VAR_BINARY_SCHEMA.getField(0));
for (byte[] key : keys) {
- // Filter now based on unsalted minMaxRange and ignore the point key salt byte
- if ( unsaltedMinMaxRange.compareLowerToUpperBound(key, offset, key.length-offset, true, comparator) <= 0 &&
- unsaltedMinMaxRange.compareUpperToLowerBound(key, offset, key.length-offset, true, comparator) >= 0) {
- keyRanges.add(KeyRange.getKeyRange(key));
- }
+ keyRanges.add(KeyRange.getKeyRange(key));
}
// while doing a point look up if after intersecting with the MinMaxrange there are
// no more keyranges left then just return
@@ -151,17 +136,11 @@ public class ScanRanges {
}
scanRange = KeyRange.getKeyRange(minKey, maxKey);
}
- if (minMaxRange != KeyRange.EVERYTHING_RANGE) {
- // Intersect using modified min/max range, but keep original range to ensure it
- // can still be decomposed into it's parts
- KeyRange inclusiveExclusiveMinMaxRange = ScanUtil.convertToInclusiveExclusiveRange(minMaxRange, schema, new ImmutableBytesWritable());
- scanRange = scanRange.intersect(inclusiveExclusiveMinMaxRange);
- }
-
+
if (scanRange == KeyRange.EMPTY_RANGE) {
return NOTHING;
}
- return new ScanRanges(schema, slotSpan, sortedRanges, scanRange, minMaxRange, useSkipScan, isPointLookup, nBuckets, rowTimestampRange);
+ return new ScanRanges(schema, slotSpan, sortedRanges, scanRange, useSkipScan, isPointLookup, nBuckets, rowTimestampRange);
}
private SkipScanFilter filter;
@@ -172,15 +151,13 @@ public class ScanRanges {
private final boolean isSalted;
private final boolean useSkipScanFilter;
private final KeyRange scanRange;
- private final KeyRange minMaxRange;
private final TimeRange rowTimestampRange;
- private ScanRanges (RowKeySchema schema, int[] slotSpan, List<List<KeyRange>> ranges, KeyRange scanRange, KeyRange minMaxRange, boolean useSkipScanFilter, boolean isPointLookup, Integer bucketNum, TimeRange rowTimestampRange) {
+ private ScanRanges (RowKeySchema schema, int[] slotSpan, List<List<KeyRange>> ranges, KeyRange scanRange, boolean useSkipScanFilter, boolean isPointLookup, Integer bucketNum, TimeRange rowTimestampRange) {
this.isPointLookup = isPointLookup;
this.isSalted = bucketNum != null;
this.useSkipScanFilter = useSkipScanFilter;
this.scanRange = scanRange;
- this.minMaxRange = minMaxRange;
this.rowTimestampRange = rowTimestampRange;
if (isSalted && !isPointLookup) {
@@ -199,14 +176,6 @@ public class ScanRanges {
}
}
- /**
- * Get the minMaxRange that is applied in addition to the scan range.
- * Only used by the ExplainTable to generate the explain plan.
- */
- public KeyRange getMinMaxRange() {
- return minMaxRange;
- }
-
public void initializeScan(Scan scan) {
scan.setStartRow(scanRange.getLowerRange());
scan.setStopRow(scanRange.getUpperRange());
@@ -580,29 +549,9 @@ public class ScanRanges {
}
public int getBoundPkColumnCount() {
- return Math.max(getBoundPkSpan(ranges, slotSpan), getBoundMinMaxSlotCount());
+ return getBoundPkSpan(ranges, slotSpan);
}
- private int getBoundMinMaxSlotCount() {
- if (minMaxRange == KeyRange.EMPTY_RANGE || minMaxRange == KeyRange.EVERYTHING_RANGE) {
- return 0;
- }
- ImmutableBytesWritable ptr = new ImmutableBytesWritable();
- // We don't track how many slots are bound for the minMaxRange, so we need
- // to traverse the upper and lower range key and count the slots.
- int lowerCount = 0;
- int maxOffset = schema.iterator(minMaxRange.getLowerRange(), ptr);
- for (int pos = 0; Boolean.TRUE.equals(schema.next(ptr, pos, maxOffset)); pos++) {
- lowerCount++;
- }
- int upperCount = 0;
- maxOffset = schema.iterator(minMaxRange.getUpperRange(), ptr);
- for (int pos = 0; Boolean.TRUE.equals(schema.next(ptr, pos, maxOffset)); pos++) {
- upperCount++;
- }
- return Math.max(lowerCount, upperCount);
- }
-
public int getBoundSlotCount() {
int count = 0;
boolean hasUnbound = false;
[4/4] phoenix git commit: PHOENIX-4790 Simplify check for client side
delete
Posted by ja...@apache.org.
PHOENIX-4790 Simplify check for client side delete
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d35a7519
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d35a7519
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d35a7519
Branch: refs/heads/master
Commit: d35a7519e65174978657b4c6254d595fac1b0009
Parents: 6195f8e
Author: James Taylor <ja...@apache.org>
Authored: Tue Jun 19 16:33:21 2018 +0200
Committer: James Taylor <ja...@apache.org>
Committed: Wed Jul 11 07:28:47 2018 -0700
----------------------------------------------------------------------
.../apache/phoenix/compile/DeleteCompiler.java | 24 ++++----------------
1 file changed, 5 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d35a7519/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 5f9c76c..78b2db9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -46,7 +46,6 @@ import org.apache.phoenix.execute.AggregatePlan;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.execute.MutationState.MultiRowMutationState;
import org.apache.phoenix.execute.MutationState.RowMutationState;
-import org.apache.phoenix.filter.SkipScanFilter;
import org.apache.phoenix.hbase.index.ValueGetter;
import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
@@ -481,6 +480,7 @@ public class DeleteCompiler {
projectedColumns.add(column);
aliasedNodes.add(FACTORY.aliasedNode(null, FACTORY.column(null, '"' + column.getName().getString() + '"', null)));
}
+ boolean noQueryReqd = true;
// Project all non PK indexed columns so that we can do the proper index maintenance
for (PTable index : table.getIndexes()) {
IndexMaintainer maintainer = index.getIndexMaintainer(table, connection);
@@ -492,6 +492,8 @@ public class DeleteCompiler {
boolean hasNoColumnFamilies = table.getColumnFamilies().isEmpty();
PColumn column = hasNoColumnFamilies ? table.getColumnForColumnName(columnName) : table.getColumnFamily(familyName).getPColumnForColumnName(columnName);
if(!projectedColumns.contains(column)) {
+ // We must run a query if any index contains a non pk column
+ noQueryReqd = false;
projectedColumns.add(column);
aliasedNodes.add(FACTORY.aliasedNode(null, FACTORY.column(hasNoColumnFamilies ? null : TableName.create(null, familyName), '"' + columnName + '"', null)));
}
@@ -511,7 +513,7 @@ public class DeleteCompiler {
select = StatementNormalizer.normalize(transformedSelect, resolverToBe);
}
final boolean hasPreOrPostProcessing = hasPreProcessing || hasPostProcessing;
- boolean noQueryReqd = !hasPreOrPostProcessing;
+ noQueryReqd &= !hasPreOrPostProcessing;
// No limit and no sub queries, joins, etc in where clause
// Can't run on same server for transactional data, as we need the row keys for the data
// that is being upserted for conflict detection purposes.
@@ -550,24 +552,8 @@ public class DeleteCompiler {
}
runOnServer &= queryPlans.get(0).getTableRef().getTable().getType() != PTableType.INDEX;
-
- // We need to have all indexed columns available in all immutable indexes in order
- // to generate the delete markers from the query. We also cannot have any filters
- // except for our SkipScanFilter for point lookups.
- // A simple check of the non existence of a where clause in the parse node is not sufficient, as the where clause
- // may have been optimized out. Instead, we check that there's a single SkipScanFilter
- // If we can generate a plan for every index, that means all the required columns are available in every index,
- // hence we can drive the delete from any of the plans.
- noQueryReqd &= queryPlans.size() == 1 + clientSideIndexes.size();
- int queryPlanIndex = 0;
- while (noQueryReqd && queryPlanIndex < queryPlans.size()) {
- QueryPlan plan = queryPlans.get(queryPlanIndex++);
- StatementContext context = plan.getContext();
- noQueryReqd &= (!context.getScan().hasFilter()
- || context.getScan().getFilter() instanceof SkipScanFilter)
- && context.getScanRanges().isPointLookup();
- }
+ noQueryReqd &= queryPlans.size() == 1 + clientSideIndexes.size();
final int maxSize = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_ATTRIB,QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
final int maxSizeBytes = services.getProps().getInt(QueryServices.MAX_MUTATION_SIZE_BYTES_ATTRIB,QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE_BYTES);