You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by td...@apache.org on 2019/05/03 21:53:12 UTC
[phoenix] branch phoenix-stats updated: Fix slotSpan appending to
schema issue in scanRanges.getRowKeyRanges()
This is an automated email from the ASF dual-hosted git repository.
tdsilva pushed a commit to branch phoenix-stats
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/phoenix-stats by this push:
new 2487d9d Fix slotSpan appending to schema issue in scanRanges.getRowKeyRanges()
new 4d4e9c8 Merge pull request #496 from dbwong/phoenix-stats
2487d9d is described below
commit 2487d9d729b12f808266bf343b1e36d2cfcd8089
Author: Daniel Wong <da...@salesforce.com>
AuthorDate: Mon Apr 29 18:58:23 2019 -0700
Fix slotSpan appending to schema issue in scanRanges.getRowKeyRanges()
---
.../org/apache/phoenix/compile/ScanRanges.java | 18 ++-
.../phoenix/compile/ScanRangesIntersectTest.java | 123 +++++++++++++++++++++
2 files changed, 137 insertions(+), 4 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
index c802678..1b695eb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java
@@ -719,13 +719,23 @@ public class ScanRanges {
int[] slotSpans = this.getSlotSpans();
- // If the ranges here do not qualify all the keys then those keys are unbound
- if (newRanges.size() < schema.getMaxFields()) {
+ int fieldSpan = 0;
+ for(int i = 0; i < slotSpans.length; i++){
+ //Account for a slot covering multiple fields
+ fieldSpan = fieldSpan + slotSpans[i] + 1;
+ }
+
+ // If the spans here do not qualify all the keys then those keys are unbound
+ int maxFields = schema.getMaxFields();
+ if (fieldSpan < maxFields) {
int originalSize = newRanges.size();
- for (int i = 0; i < schema.getMaxFields() - originalSize; i++) {
+ for (int i = 0; i < maxFields - originalSize; i++) {
newRanges.add(Lists.newArrayList(KeyRange.EVERYTHING_RANGE));
}
- slotSpans = new int[schema.getMaxFields()];
+ //extend slots
+ int slotsToAdd = (maxFields - fieldSpan);
+ int newSlotLength = slotSpans.length + slotsToAdd;
+ slotSpans = new int[newSlotLength];
System.arraycopy(this.getSlotSpans(), 0, slotSpans, 0, this.getSlotSpans().length);
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java
index accc28e..df1bade 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java
@@ -36,6 +36,9 @@ import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PDouble;
+import org.apache.phoenix.schema.types.PSmallint;
+import org.apache.phoenix.schema.types.PUnsignedTinyint;
import org.apache.phoenix.schema.types.PVarchar;
import org.junit.Test;
@@ -112,6 +115,33 @@ public class ScanRangesIntersectTest {
}
};
+ private static PDatum SIMPLE_TINYINT = new PDatum() {
+ @Override
+ public boolean isNullable() {
+ return false;
+ }
+
+ @Override
+ public PDataType getDataType() {
+ return PUnsignedTinyint.INSTANCE;
+ }
+
+ @Override
+ public Integer getMaxLength() {
+ return 1;
+ }
+
+ @Override
+ public Integer getScale() {
+ return null;
+ }
+
+ @Override
+ public SortOrder getSortOrder() {
+ return SortOrder.getDefault();
+ }
+ };
+
// Does not handle some edge conditions like empty string
private String handleScanNextKey(String key) {
char lastChar = key.charAt(key.length() - 1);
@@ -485,6 +515,96 @@ public class ScanRangesIntersectTest {
assertEquals(singleKeyToScanRange("BD"),rowKeyRanges.get(3).toString());
}
+ @Test
+ public void getRowKeyRangesMultipleFieldsSingleSlot() {
+ int rowKeySchemaFields = 3;
+ RowKeySchema schema = buildSimpleRowKeySchema(rowKeySchemaFields);
+
+ int[] slotSpan = new int[1];
+ slotSpan[0] = 2;
+
+ KeyRange keyRange1 = KeyRange.getKeyRange(stringToByteArray("ABC"));
+ KeyRange keyRange2 = KeyRange.getKeyRange(stringToByteArray("DEF"));
+ List<List<KeyRange>> ranges = new ArrayList<>();
+ ranges.add(Lists.newArrayList(keyRange1,keyRange2));
+
+ ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1);
+
+ List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges();
+ assertEquals(2, rowKeyRanges.size());
+ assertEquals(singleKeyToScanRange("ABC"),rowKeyRanges.get(0).toString());
+ assertEquals(singleKeyToScanRange("DEF"),rowKeyRanges.get(1).toString());
+ }
+
+ @Test
+ public void getRowKeyRangesMultipleFieldsFrontLoadedSlot() {
+ int rowKeySchemaFields = 3;
+ RowKeySchema schema = buildSimpleRowKeySchema(rowKeySchemaFields);
+
+ int[] slotSpan = new int[2];
+ slotSpan[0] = 1;
+
+ KeyRange keyRange1 = KeyRange.getKeyRange(stringToByteArray("AB"));
+ List<List<KeyRange>> ranges = new ArrayList<>();
+ ranges.add(Lists.newArrayList(keyRange1));
+
+ KeyRange keyRange3 = KeyRange.getKeyRange(stringToByteArray("C"));
+ ranges.add(Lists.newArrayList(keyRange3));
+
+ ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1);
+
+ List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges();
+ assertEquals(1, rowKeyRanges.size());
+ assertEquals("[ABC - ABD)",rowKeyRanges.get(0).toString());
+ }
+
+ @Test
+ public void getRowKeyRangesMultipleFieldsBackLoadedSlot() {
+ int rowKeySchemaFields = 3;
+ RowKeySchema schema = buildSimpleRowKeySchema(rowKeySchemaFields);
+
+ int[] slotSpan = new int[2];
+ slotSpan[1] = 1;
+
+ KeyRange keyRange1 = KeyRange.getKeyRange(stringToByteArray("A"));
+ List<List<KeyRange>> ranges = new ArrayList<>();
+ ranges.add(Lists.newArrayList(keyRange1));
+
+ KeyRange keyRange3 = KeyRange.getKeyRange(stringToByteArray("BC"));
+ ranges.add(Lists.newArrayList(keyRange3));
+
+ ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1);
+
+ List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges();
+ assertEquals(1, rowKeyRanges.size());
+ assertEquals("[ABC - ABD)",rowKeyRanges.get(0).toString());
+ }
+
+ @Test
+ public void getRowKeyRangesUpperBoundOverflows() {
+ int rowKeySchemaFields = 2;
+ RowKeySchema.RowKeySchemaBuilder builder = new RowKeySchema.RowKeySchemaBuilder(rowKeySchemaFields);
+ for(int i = 0; i < rowKeySchemaFields; i++) {
+ builder.addField(SIMPLE_TINYINT, SIMPLE_TINYINT.isNullable(), SIMPLE_TINYINT.getSortOrder());
+ }
+ RowKeySchema schema = builder.build();
+
+ int[] slotSpan = new int[2];
+
+ KeyRange keyRange1 = KeyRange.getKeyRange(new byte[]{-1});
+ List<List<KeyRange>> ranges = new ArrayList<>();
+ ranges.add(Lists.newArrayList(keyRange1));
+
+ KeyRange keyRange3 = KeyRange.getKeyRange(new byte[]{-1});
+ ranges.add(Lists.newArrayList(keyRange3));
+
+ ScanRanges scanRanges = ScanRanges.create(schema, ranges, slotSpan, null, true, -1);
+
+ List<KeyRange> rowKeyRanges = scanRanges.getRowKeyRanges();
+ assertEquals(1, rowKeyRanges.size());
+ assertEquals("[\\xFF\\xFF - *)",rowKeyRanges.get(0).toString());
+ }
+
private RowKeySchema buildSimpleRowKeySchema(int fields){
RowKeySchema.RowKeySchemaBuilder builder = new RowKeySchema.RowKeySchemaBuilder(fields);
for(int i = 0; i < fields; i++) {
@@ -494,6 +614,9 @@ public class ScanRangesIntersectTest {
}
private String singleKeyToScanRange(String key){
+ //Appending 0 byte is next key for variable length asc fields
+ //This includes point gets for more than 1 key as ScanRanges treats the combined pk
+ // as a varbinary
return String.format("[%s - %s\\x00)",key,key);
}
}