You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by la...@apache.org on 2020/09/09 23:31:36 UTC
[phoenix] branch master updated: PHOENIX-5171 SkipScan incorrectly
filters composite primary key which the key range contains all values.
This is an automated email from the ASF dual-hosted git repository.
larsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new f5cc0ad PHOENIX-5171 SkipScan incorrectly filters composite primary key which the key range contains all values.
f5cc0ad is described below
commit f5cc0ad49f13283fefccddf1b187da95eecdb423
Author: Lars <la...@apache.org>
AuthorDate: Wed Sep 9 16:32:03 2020 -0700
PHOENIX-5171 SkipScan incorrectly filters composite primary key which the key range contains all values.
---
.../apache/phoenix/end2end/SkipScanQueryIT.java | 33 ++++++++++++++++++++
.../java/org/apache/phoenix/util/ScanUtil.java | 5 +--
.../apache/phoenix/filter/SkipScanFilterTest.java | 36 +++++++++++++++++++++-
3 files changed, 69 insertions(+), 5 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
index 64b897dd..6e269da 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipScanQueryIT.java
@@ -713,4 +713,37 @@ public class SkipScanQueryIT extends ParallelStatsDisabledIT {
assertResultSet(rs, new Object[][]{{1,3,2,10},{1,3,5,6}});
}
}
+
+ @Test
+ public void testKeyRangesContainsAllValues() throws Exception {
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE IF NOT EXISTS " + tableName + "(" +
+ " vdate VARCHAR, " +
+ " tab VARCHAR, " +
+ " dev TINYINT NOT NULL, " +
+ " app VARCHAR, " +
+ " target VARCHAR, " +
+ " channel VARCHAR, " +
+ " one VARCHAR, " +
+ " two VARCHAR, " +
+ " count1 INTEGER, " +
+ " count2 INTEGER, " +
+ " CONSTRAINT PK PRIMARY KEY (vdate,tab,dev,app,target,channel,one,two))";
+
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute(ddl);
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('2018-02-14','channel_agg',2,null,null,'A004',null,null,2,2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('2018-02-14','channel_agg',2,null,null,null,null,null,2,2)");
+ conn.commit();
+
+ ResultSet rs = conn.createStatement().executeQuery(
+ "SELECT * FROM " + tableName +
+ " WHERE dev = 2 AND vdate BETWEEN '2018-02-10' AND '2019-02-19'" +
+ " AND tab = 'channel_agg' AND channel='A004'");
+
+ assertTrue(rs.next());
+ assertEquals("2018-02-14", rs.getString(1));
+ assertFalse(rs.next());
+ }
+ }
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 0a37411..c892ed2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -406,12 +406,9 @@ public class ScanUtil {
* for the same reason. However, if the type is variable width
* continue building the key because null values will be filtered
* since our separator byte will be appended and incremented.
- * 3) if the range includes everything as we cannot add any more useful
- * information to the key after that.
*/
lastUnboundUpper = false;
- if ( range.isUnbound(bound) &&
- ( bound == Bound.UPPER || isFixedWidth || range == KeyRange.EVERYTHING_RANGE) ){
+ if (range.isUnbound(bound) && (bound == Bound.UPPER || isFixedWidth)) {
lastUnboundUpper = (bound == Bound.UPPER);
break;
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
index 8f78588..641db78 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanFilterTest.java
@@ -138,7 +138,8 @@ public class SkipScanFilterTest extends TestCase {
QueryConstants.SEPARATOR_BYTE_ARRAY,
Bytes.toBytes("1") ),
ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
- Bytes.toBytes("b") )),
+ Bytes.toBytes("b"),QueryConstants.SEPARATOR_BYTE_ARRAY,
+ QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("1") )),
new Include(ByteUtil.concat(Bytes.toBytes("f"),QueryConstants.SEPARATOR_BYTE_ARRAY,
Bytes.toBytes("b"), QueryConstants.SEPARATOR_BYTE_ARRAY,
QueryConstants.SEPARATOR_BYTE_ARRAY,
@@ -146,6 +147,39 @@ public class SkipScanFilterTest extends TestCase {
);
testCases.addAll(
foreach(new KeyRange[][]{{
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("2018-02-10"), true, Bytes.toBytes("2019-02-19"), true),
+ },
+ {
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("channel"), true, Bytes.toBytes("channel"), true),
+ },
+ {
+ PChar.INSTANCE.getKeyRange(Bytes.toBytes("2"), true, Bytes.toBytes("2"), true),
+ },
+ {
+ KeyRange.EVERYTHING_RANGE,
+ },
+ {
+ KeyRange.EVERYTHING_RANGE,
+ },
+ {
+ PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("A004"), true, Bytes.toBytes("A004"), true),
+ },
+ },
+ new int[] {0, 0, 1, 0, 0, 0, 0, 0},
+ null,
+ new SeekNext(
+ ByteUtil.concat(Bytes.toBytes("2018-02-14"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("channel"),
+ QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("2")),
+ ByteUtil.concat(Bytes.toBytes("2018-02-14"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("channel"),
+ QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("2"), QueryConstants.SEPARATOR_BYTE_ARRAY, QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("A004"))),
+ new Include(ByteUtil.concat(Bytes.toBytes("2018-02-15"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("channel"),
+ QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("2"), QueryConstants.SEPARATOR_BYTE_ARRAY, QueryConstants.SEPARATOR_BYTE_ARRAY,
+ Bytes.toBytes("A004")))
+ )
+ );
+ testCases.addAll(
+ foreach(new KeyRange[][]{{
PVarchar.INSTANCE.getKeyRange(Bytes.toBytes("20160116121006"), true, Bytes.toBytes("20160116181006"), true),
},
{