You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by la...@apache.org on 2016/06/21 23:10:03 UTC
phoenix git commit: PHOENIX-3012 DistinctPrefixFilter logic fails
with local indexes and salted tables.
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-0.98 f043275d3 -> 440219dfd
PHOENIX-3012 DistinctPrefixFilter logic fails with local indexes and salted tables.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/440219df
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/440219df
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/440219df
Branch: refs/heads/4.x-HBase-0.98
Commit: 440219dfde574b6351af491a11082fdeca8e57ce
Parents: f043275
Author: Lars Hofhansl <la...@apache.org>
Authored: Tue Jun 21 15:59:19 2016 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Tue Jun 21 15:59:19 2016 -0700
----------------------------------------------------------------------
.../phoenix/end2end/DistinctPrefixFilterIT.java | 2 +-
.../phoenix/filter/DistinctPrefixFilter.java | 55 ++++++++++++++------
.../java/org/apache/phoenix/util/ScanUtil.java | 4 ++
3 files changed, 44 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/440219df/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java
index d6640ae..203d51e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DistinctPrefixFilterIT.java
@@ -60,7 +60,7 @@ public class DistinctPrefixFilterIT extends BaseHBaseManagedTimeTableReuseIT {
createTestTable(getUrl(), ddl);
conn.prepareStatement("CREATE INDEX " + testTableF + "_idx ON "+testTableF+"(col2) DISABLE_WAL=true").execute();
- conn.prepareStatement("CREATE INDEX " + testTableV + "_idx ON "+testTableV+"(col2) DISABLE_WAL=true").execute();
+ conn.prepareStatement("CREATE LOCAL INDEX " + testTableV + "_idx ON "+testTableV+"(col2) DISABLE_WAL=true").execute();
conn.prepareStatement("CREATE SEQUENCE " + testSeq + " CACHE 1000").execute();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/440219df/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java
index a3e8a0a..a4cbe8a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/DistinctPrefixFilter.java
@@ -36,6 +36,7 @@ import org.apache.phoenix.util.ByteUtil;
public class DistinctPrefixFilter extends FilterBase implements Writable {
private static byte VERSION = 1;
+ private int offset;
private RowKeySchema schema;
private int prefixLengh;
private boolean filterAll = false;
@@ -49,12 +50,16 @@ public class DistinctPrefixFilter extends FilterBase implements Writable {
this.prefixLengh = prefixLength;
}
+ public void setOffset(int offset) {
+ this.offset = offset;
+ }
+
@Override
public ReturnCode filterKeyValue(Cell v) throws IOException {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
// First determine the prefix based on the schema
- int maxOffset = schema.iterator(v.getRowArray(), v.getRowOffset(), v.getRowLength(), ptr);
+ int maxOffset = schema.iterator(v.getRowArray(), v.getRowOffset()+offset, v.getRowLength()-offset, ptr);
schema.next(ptr, 0, maxOffset, prefixLengh - 1);
// now check whether we have seen this prefix before
@@ -70,26 +75,44 @@ public class DistinctPrefixFilter extends FilterBase implements Writable {
@Override
public Cell getNextCellHint(Cell v) throws IOException {
- ImmutableBytesWritable tmp;
PDataType<?> type = schema.getField(prefixLengh-1).getDataType();
- if (reversed) {
- // simply seek right before the first occurrence of the row
- tmp = lastKey;
+
+ ImmutableBytesWritable tmp;
+ // In the following we make sure we copy the key at most once
+ // Either because we have an offset, or when needed for nextKey
+ if (offset > 0) {
+ // make space to copy the missing offset, also 0-pad here if needed
+ // (since we're making a copy anyway)
+ byte[] tmpKey = new byte[offset + lastKey.getLength() +
+ (reversed || type.isFixedWidth() ? 0 : 1)];
+ System.arraycopy(v.getRowArray(), v.getRowOffset(), tmpKey, 0, offset);
+ System.arraycopy(lastKey.get(), 0, tmpKey, offset, lastKey.getLength());
+ tmp = new ImmutableBytesWritable(tmpKey);
+ if (!reversed) {
+ // calculate the next key, the above already 0-padded if needed
+ if (!ByteUtil.nextKey(tmp.get(), tmp.getOffset(), tmp.getLength())) {
+ filterAll = true;
+ }
+ }
} else {
- if (type.isFixedWidth()) {
- // copy the bytes, since nextKey will modify in place
- tmp = new ImmutableBytesWritable(lastKey.copyBytes());
+ if (reversed) {
+ // simply seek right before the first occurrence of the row
+ tmp = lastKey;
} else {
- // pad with a 0x00 byte (makes a copy)
- tmp = new ImmutableBytesWritable(lastKey);
- ByteUtil.nullPad(tmp, tmp.getLength() + 1);
- }
- // calculate the next key
- if (!ByteUtil.nextKey(tmp.get(), tmp.getOffset(), tmp.getLength())) {
- filterAll = true;
+ if (type.isFixedWidth()) {
+ // copy the bytes, since nextKey will modify in place
+ tmp = new ImmutableBytesWritable(lastKey.copyBytes());
+ } else {
+ // pad with a 0x00 byte (makes a copy)
+ tmp = new ImmutableBytesWritable(lastKey);
+ ByteUtil.nullPad(tmp, tmp.getLength() + 1);
+ }
+ // calculate the next key
+ if (!ByteUtil.nextKey(tmp.get(), tmp.getOffset(), tmp.getLength())) {
+ filterAll = true;
+ }
}
}
-
return KeyValue.createFirstOnRow(tmp.get(), tmp.getOffset(), tmp.getLength(), null, 0, 0,
null, 0, 0);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/440219df/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 7a3014b..711717a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -53,6 +53,7 @@ import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.execute.DescVarLengthFastByteComparisons;
import org.apache.phoenix.filter.BooleanExpressionFilter;
+import org.apache.phoenix.filter.DistinctPrefixFilter;
import org.apache.phoenix.filter.SkipScanFilter;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.KeyRange;
@@ -693,6 +694,9 @@ public class ScanUtil {
} else if (filter instanceof SkipScanFilter) {
SkipScanFilter skipScanFilter = (SkipScanFilter)filter;
skipScanFilter.setOffset(offset);
+ } else if (filter instanceof DistinctPrefixFilter) {
+ DistinctPrefixFilter prefixFilter = (DistinctPrefixFilter) filter;
+ prefixFilter.setOffset(offset);
}
}