You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2015/12/14 20:48:01 UTC
[3/4] phoenix git commit: PHOENIX-2455 Partial results for a query
when PHOENIX-2194 is applied
PHOENIX-2455 Partial results for a query when PHOENIX-2194 is applied
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9f82aaa1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9f82aaa1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9f82aaa1
Branch: refs/heads/4.x-HBase-0.98
Commit: 9f82aaa1399200de560958ff9be85bc5c43b3132
Parents: 0790f56
Author: James Taylor <jt...@salesforce.com>
Authored: Sun Dec 13 14:32:29 2015 -0800
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Dec 14 11:46:17 2015 -0800
----------------------------------------------------------------------
.../apache/phoenix/end2end/SkipScanQueryIT.java | 31 +++++++++
.../apache/phoenix/filter/SkipScanFilter.java | 3 +-
.../java/org/apache/phoenix/query/KeyRange.java | 48 +++++++++-----
.../java/org/apache/phoenix/util/ScanUtil.java | 2 +-
.../phoenix/filter/SkipScanFilterTest.java | 66 +++++++++++++-------
5 files changed, 108 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9f82aaa1/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
index 1937f65..2ade0a4 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
@@ -110,6 +110,37 @@ public class SkipScanQueryIT extends BaseHBaseManagedTimeIT {
}
@Test
+ public void testSkipScanFilterQuery() throws Exception {
+ String createTableDDL = "CREATE TABLE test" + "(col1 VARCHAR," + "col2 VARCHAR," + "col3 VARCHAR,"
+ + "col4 VARCHAR," + "CONSTRAINT pk " + "PRIMARY KEY (col1,col2,col3,col4))";
+ String upsertQuery = "upsert into test values(?,?,?,?)";
+ String query = "SELECT col1, col2, col3, col4 FROM test WHERE col1 IN ('a','e','f') AND col2 = 'b' AND col4 = '1' ";
+ String[] col1Values = { "a", "e.f", "f" };
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ createTestTable(getUrl(), createTableDDL);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(true);
+ try {
+ PreparedStatement statement = conn.prepareStatement(upsertQuery);
+ for (String col1Value : col1Values) {
+ statement.setString(1, col1Value);
+ statement.setString(2, "b");
+ statement.setString(3, "");
+ statement.setString(4, "1");
+ statement.execute();
+ }
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "a");
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "f");
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
public void testSelectAfterUpsertInQuery() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
initSelectAfterUpsertTable(conn);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9f82aaa1/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
index d1c8532..667b3d7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SkipScanFilter.java
@@ -580,8 +580,7 @@ public class SkipScanFilter extends FilterBase implements Writable {
List<KeyRange> orClause = Lists.newArrayListWithExpectedSize(orLen);
slots.add(orClause);
for (int j=0; j<orLen; j++) {
- KeyRange range = new KeyRange();
- range.readFields(in);
+ KeyRange range = KeyRange.read(in);
orClause.add(range);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9f82aaa1/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
index 6618aa5..f4bf793 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/KeyRange.java
@@ -119,15 +119,30 @@ public class KeyRange implements Writable {
return getKeyRange(lowerRange, true, upperRange, false);
}
- // TODO: make non public and move to org.apache.phoenix.type soon
- public static KeyRange getKeyRange(byte[] lowerRange, boolean lowerInclusive,
+ private static KeyRange getSingleton(byte[] lowerRange, boolean lowerInclusive,
byte[] upperRange, boolean upperInclusive) {
if (lowerRange == null || upperRange == null) {
return EMPTY_RANGE;
}
- // Need to treat null differently for a point range
- if (lowerRange.length == 0 && upperRange.length == 0 && lowerInclusive && upperInclusive) {
- return IS_NULL_RANGE;
+ if (lowerRange.length == 0 && upperRange.length == 0) {
+ // Need singleton to represent NULL range so it gets treated differently
+ // than an unbound RANGE.
+ return lowerInclusive && upperInclusive ? IS_NULL_RANGE : EVERYTHING_RANGE;
+ }
+ if (lowerRange.length != 0 && upperRange.length != 0) {
+ int cmp = Bytes.compareTo(lowerRange, upperRange);
+ if (cmp > 0 || (cmp == 0 && !(lowerInclusive && upperInclusive))) {
+ return EMPTY_RANGE;
+ }
+ }
+ return null;
+ }
+
+ public static KeyRange getKeyRange(byte[] lowerRange, boolean lowerInclusive,
+ byte[] upperRange, boolean upperInclusive) {
+ KeyRange range = getSingleton(lowerRange, lowerInclusive, upperRange, upperInclusive);
+ if (range != null) {
+ return range;
}
boolean unboundLower = false;
boolean unboundUpper = false;
@@ -142,20 +157,23 @@ public class KeyRange implements Writable {
unboundUpper = true;
}
- if (unboundLower && unboundUpper) {
- return EVERYTHING_RANGE;
- }
- if (!unboundLower && !unboundUpper) {
- int cmp = Bytes.compareTo(lowerRange, upperRange);
- if (cmp > 0 || (cmp == 0 && !(lowerInclusive && upperInclusive))) {
- return EMPTY_RANGE;
- }
- }
return new KeyRange(lowerRange, unboundLower ? false : lowerInclusive,
upperRange, unboundUpper ? false : upperInclusive);
}
- public KeyRange() {
+ public static KeyRange read(DataInput input) throws IOException {
+ KeyRange range = new KeyRange();
+ range.readFields(input);
+ // Translate to singleton after reading
+ KeyRange singletonRange = getSingleton(range.lowerRange, range.lowerInclusive, range.upperRange, range.upperInclusive);
+ if (singletonRange != null) {
+ return singletonRange;
+ }
+ // Otherwise, just keep the range we read
+ return range;
+ }
+
+ private KeyRange() {
this.lowerRange = DEGENERATE_KEY;
this.lowerInclusive = false;
this.upperRange = DEGENERATE_KEY;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9f82aaa1/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 912dd03..b6a2c85 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -375,7 +375,7 @@ public class ScanUtil {
* incrementing the key value itself, and thus bumping it up too much.
*/
boolean inclusiveUpper = range.isUpperInclusive() && bound == Bound.UPPER;
- boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER;
+ boolean exclusiveLower = !range.isLowerInclusive() && bound == Bound.LOWER && range != KeyRange.EVERYTHING_RANGE;
boolean exclusiveUpper = !range.isUpperInclusive() && bound == Bound.UPPER;
// If we are setting the upper bound of using inclusive single key, we remember
// to increment the key if we exit the loop after this iteration.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9f82aaa1/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
index 7e68e25..898f778 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
@@ -23,19 +23,17 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import junit.framework.TestCase;
-
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.PDatum;
-import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder;
import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.util.ByteUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -45,6 +43,8 @@ import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
+import junit.framework.TestCase;
+
//reset()
//filterAllRemaining() -> true indicates scan is over, false, keep going on.
//filterRowKey(byte[],int,int) -> true to drop this row, if false, we will also call
@@ -97,7 +97,6 @@ public class SkipScanFilterTest extends TestCase {
@Test
public void test() throws IOException {
- System.out.println("CNF: " + cnf + "\n" + "Expectations: " + expectations);
for (Expectation expectation : expectations) {
expectation.examine(skipper);
}
@@ -106,6 +105,39 @@ public class SkipScanFilterTest extends TestCase {
@Parameters(name="{0} {1} {2}")
public static Collection<Object> data() {
List<Object> testCases = Lists.newArrayList();
+ // Variable length tests
+ testCases.addAll(
+ foreach(new KeyRange[][]{{
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("a"), true, Bytes.toBytes("a"), true),
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("e"), true, Bytes.toBytes("e"), true),
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("f"), true, Bytes.toBytes("f"), true),
+ },
+ {
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("b"), true, Bytes.toBytes("b"), true),
+ },
+ {
+ KeyRange.EVERYTHING_RANGE,
+ },
+ {
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("1"), true, Bytes.toBytes("1"), true),
+ }},
+ new int[4],
+ new Include(ByteUtil.concat(Bytes.toBytes("a"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("1") ) ),
+ new SeekNext(ByteUtil.concat(Bytes.toBytes("e.f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("1") ),
+ ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b") )),
+ new Include(ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("1") ) ) )
+ );
+ // Fixed length tests
testCases.addAll(
foreach(new KeyRange[][]{{
PChar.INSTANCE.getKeyRange(Bytes.toBytes("abc"), true, Bytes.toBytes("def"), true),
@@ -124,7 +156,6 @@ public class SkipScanFilterTest extends TestCase {
PChar.INSTANCE.getKeyRange(Bytes.toBytes("AA"), true, Bytes.toBytes("AB"), false),
}},
new int[]{3,2,2,2,2},
- //new SeekNext("abcABABABAB", "abdAAAAAAAA"),
new SeekNext("defAAABABAB", "dzzAAAAAAAA"),
new Finished("xyyABABABAB"))
);
@@ -309,23 +340,6 @@ public class SkipScanFilterTest extends TestCase {
new SeekNext("dzzAB250", "dzzAB701"),
new Finished("zzzAA000"))
);
-// TODO variable length columns
-// testCases.addAll(
-// foreach(new KeyRange[][]{{
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("apple"), true, Bytes.toBytes("lemon"), true),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("pear"), false, Bytes.toBytes("yam"), false),
-// },
-// {
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("AB"), true, Bytes.toBytes("AX"), true),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("EA"), false, Bytes.toBytes("EZ"), false),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("PO"), true, Bytes.toBytes("PP"), false),
-// },
-// {
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("100"), true, Bytes.toBytes("250"), false),
-// Char.INSTANCE.getKeyRange(Bytes.toBytes("700"), false, Bytes.toBytes("901"), false),
-// }},
-// new int[]{3,3})
-// );
return testCases;
}
@@ -378,6 +392,10 @@ public class SkipScanFilterTest extends TestCase {
this.rowkey = Bytes.toBytes(rowkey);
}
+ public Include(byte[] rowkey) {
+ this.rowkey = rowkey;
+ }
+
@SuppressWarnings("deprecation")
@Override public void examine(SkipScanFilter skipper) throws IOException {
KeyValue kv = KeyValue.createFirstOnRow(rowkey);