You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2024/07/10 08:40:52 UTC
(hbase) branch branch-3 updated: HBASE-28717 Support FuzzyRowFilter in REST interface (#6060)
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 282f0c0c994 HBASE-28717 Support FuzzyRowFilter in REST interface (#6060)
282f0c0c994 is described below
commit 282f0c0c9941734159355b6c8d924d9079815ac0
Author: Istvan Toth <st...@apache.org>
AuthorDate: Wed Jul 10 10:39:33 2024 +0200
HBASE-28717 Support FuzzyRowFilter in REST interface (#6060)
Signed-off-by: Ankit Singhal <an...@apache.org>
(cherry picked from commit 429d6bab62a7926afc6e84f282f15cab39598725)
---
.../apache/hadoop/hbase/filter/FuzzyRowFilter.java | 27 ++++++++
.../hadoop/hbase/rest/model/ScannerModel.java | 78 +++++++++++++++++++++-
.../hadoop/hbase/rest/TestScannersWithFilters.java | 25 ++++++-
3 files changed, 128 insertions(+), 2 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index 244dc9c7ca5..f8c35b46528 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -161,6 +161,33 @@ public class FuzzyRowFilter extends FilterBase implements HintingFilter {
return true;
}
+ /**
+ * Returns the Fuzzy keys in the format expected by the constructor.
+ * @return the Fuzzy keys in the format expected by the constructor
+ */
+ public List<Pair<byte[], byte[]>> getFuzzyKeys() {
+ List<Pair<byte[], byte[]>> returnList = new ArrayList<>(fuzzyKeysData.size());
+ for (Pair<byte[], byte[]> fuzzyKey : fuzzyKeysData) {
+ Pair<byte[], byte[]> returnKey = new Pair<>();
+ // This won't revert the original key's don't care values, but we don't care.
+ returnKey.setFirst(Arrays.copyOf(fuzzyKey.getFirst(), fuzzyKey.getFirst().length));
+ byte[] returnMask = Arrays.copyOf(fuzzyKey.getSecond(), fuzzyKey.getSecond().length);
+ if (UNSAFE_UNALIGNED && isPreprocessedMask(returnMask)) {
+ // Revert the preprocessing.
+ for (int i = 0; i < returnMask.length; i++) {
+ if (returnMask[i] == -1) {
+ returnMask[i] = 0; // -1 >> 0
+ } else if (returnMask[i] == 2) {
+ returnMask[i] = 1;// 2 >> 1
+ }
+ }
+ }
+ returnKey.setSecond(returnMask);
+ returnList.add(returnKey);
+ }
+ return returnList;
+ }
+
@Override
public void reset() throws IOException {
filterRow = false;
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
index a18a9ba427a..1ccb541c887 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.hadoop.hbase.filter.MultiRowRangeFilter;
@@ -70,6 +71,7 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler;
import org.apache.hadoop.hbase.rest.RestUtil;
import org.apache.hadoop.hbase.security.visibility.Authorizations;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider;
@@ -279,6 +281,63 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
}
+ static class FuzzyKeyModel {
+
+ protected byte[] key;
+
+ protected byte[] mask;
+
+ public FuzzyKeyModel() {
+ }
+
+ public FuzzyKeyModel(Pair<byte[], byte[]> keyWithMask) {
+ this.key = keyWithMask.getFirst();
+ this.mask = keyWithMask.getSecond();
+ }
+
+ public Pair<byte[], byte[]> build() {
+ return new Pair<>(key, mask);
+ }
+
+ public byte[] getKey() {
+ return key;
+ }
+
+ public void setKey(byte[] key) {
+ this.key = key;
+ }
+
+ public byte[] getMask() {
+ return mask;
+ }
+
+ public void setMask(byte[] mask) {
+ this.mask = mask;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(key);
+ result = prime * result + Arrays.hashCode(mask);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof FuzzyKeyModel)) {
+ return false;
+ }
+ FuzzyKeyModel other = (FuzzyKeyModel) obj;
+ return Arrays.equals(key, other.key) && Arrays.equals(mask, other.mask);
+ }
+
+ }
+
// A grab bag of fields, would have been a union if this were C.
// These are null by default and will only be serialized if set (non null).
@XmlAttribute
@@ -321,6 +380,8 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
private List<RowRangeModel> ranges;
@XmlElement
public List<Long> timestamps;
+ @XmlElement
+ private List<FuzzyKeyModel> fuzzyKeys;
static enum FilterType {
ColumnCountGetFilter,
@@ -345,7 +406,8 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
SkipFilter,
TimestampsFilter,
ValueFilter,
- WhileMatchFilter
+ WhileMatchFilter,
+ FuzzyRowFilter
}
public FilterModel() {
@@ -458,6 +520,12 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
this.filters = new ArrayList<>();
this.filters.add(new FilterModel(((WhileMatchFilter) filter).getFilter()));
break;
+ case FuzzyRowFilter:
+ this.fuzzyKeys = new ArrayList<>(((FuzzyRowFilter) filter).getFuzzyKeys().size());
+ for (Pair<byte[], byte[]> keyWithMask : ((FuzzyRowFilter) filter).getFuzzyKeys()) {
+ this.fuzzyKeys.add(new FuzzyKeyModel(keyWithMask));
+ }
+ break;
default:
throw new RuntimeException("unhandled filter type " + type);
}
@@ -569,6 +637,14 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
case WhileMatchFilter:
filter = new WhileMatchFilter(filters.get(0).build());
break;
+ case FuzzyRowFilter: {
+ ArrayList<Pair<byte[], byte[]>> fuzzyKeyArgs = new ArrayList<>(fuzzyKeys.size());
+ for (FuzzyKeyModel keyModel : fuzzyKeys) {
+ fuzzyKeyArgs.add(keyModel.build());
+ }
+ filter = new FuzzyRowFilter(fuzzyKeyArgs);
+ }
+ break;
default:
throw new RuntimeException("unhandled filter type: " + type);
}
diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
index 865a6a18cd0..4b32e8a4c22 100644
--- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
+++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannersWithFilters.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
import org.apache.hadoop.hbase.filter.MultiRowRangeFilter;
import org.apache.hadoop.hbase.filter.PageFilter;
@@ -71,6 +72,7 @@ import org.apache.hadoop.hbase.rest.model.ScannerModel;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RestTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
@@ -230,7 +232,7 @@ public class TestScannersWithFilters {
int rows = cells.getRows().size();
assertEquals(
- "Scanned too many rows! Only expected " + expectedRows + " total but scanned " + rows,
+ "Unexpected number of rows! Expected " + expectedRows + " total but scanned " + rows,
expectedRows, rows);
for (RowModel row : cells.getRows()) {
int count = row.getCells().size();
@@ -978,4 +980,25 @@ public class TestScannersWithFilters {
s.setFilter(new MultiRowRangeFilter(ranges));
verifyScan(s, expectedRows, expectedKeys);
}
+
+ @Test
+ public void testFuzzyRowFilter() throws Exception {
+ long expectedRows = 4;
+ long expectedKeys = colsPerRow;
+ List<Pair<byte[], byte[]>> fuzzyKeys = new ArrayList<>();
+
+ // Exact match for ROWS_ONE[0] (one row)
+ byte[] rowOneMask = new byte[ROWS_ONE[0].length];
+ Arrays.fill(rowOneMask, (byte) 0);
+ fuzzyKeys.add(new Pair<>(ROWS_ONE[0], rowOneMask));
+ // All ROW_TWO keys (three rows)
+ byte[] rowTwoMask = new byte[ROWS_TWO[0].length];
+ Arrays.fill(rowTwoMask, (byte) 0);
+ rowTwoMask[rowTwoMask.length - 1] = (byte) 1;
+ fuzzyKeys.add(new Pair<>(ROWS_TWO[2], rowTwoMask));
+
+ Scan s = new Scan();
+ s.setFilter(new FuzzyRowFilter(fuzzyKeys));
+ verifyScan(s, expectedRows, expectedKeys);
+ }
}