You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/06/20 01:12:15 UTC
git commit: HBASE-9345 Add support for specifying filters in scan
(Virag Kothari)
Repository: hbase
Updated Branches:
refs/heads/master 9829bb9c2 -> b5db14328
HBASE-9345 Add support for specifying filters in scan (Virag Kothari)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b5db1432
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b5db1432
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b5db1432
Branch: refs/heads/master
Commit: b5db1432806ad49a04ea7866856e7321ea252adb
Parents: 9829bb9
Author: Michael Stack <st...@apache.org>
Authored: Thu Jun 19 16:11:52 2014 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Thu Jun 19 16:11:52 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/rest/Constants.java | 2 +
.../apache/hadoop/hbase/rest/RESTServlet.java | 17 +++
.../apache/hadoop/hbase/rest/TableResource.java | 20 +++-
.../apache/hadoop/hbase/rest/TestTableScan.java | 108 ++++++++++++++++++-
4 files changed, 144 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5db1432/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
index 4326c41..c3c8796 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
@@ -68,6 +68,8 @@ public interface Constants {
String SCAN_BATCH_SIZE = "batchsize";
String SCAN_LIMIT = "limit";
String SCAN_FETCH_SIZE = "hbase.rest.scan.fetchsize";
+ String SCAN_FILTER = "filter";
+ String CUSTOM_FILTERS = "hbase.rest.custom.filters";
String ROW_KEYS_PARAM_NAME = "row";
/** If this query parameter is present when processing row or scanner resources,
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5db1432/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
index c265e40..4f0bb89 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -186,6 +187,7 @@ public class RESTServlet implements Constants {
this.realUser = realUser;
this.conf = conf;
+ registerCustomFilter(conf);
}
/**
@@ -255,4 +257,19 @@ public class RESTServlet implements Constants {
}
return connInfo;
}
+
+ private void registerCustomFilter(Configuration conf) {
+ String[] filterList = conf.getStrings(Constants.CUSTOM_FILTERS);
+ if (filterList != null) {
+ for (String filterClass : filterList) {
+ String[] filterPart = filterClass.split(":");
+ if (filterPart.length != 2) {
+ LOG.warn(
+ "Invalid filter specification " + filterClass + " - skipping");
+ } else {
+ ParseFilter.registerFilter(filterPart[0], filterPart[1]);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5db1432/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
index 963e900..0627ed2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.util.Bytes;
@@ -130,7 +132,8 @@ public class TableResource extends ResourceBase {
@DefaultValue("-1") @QueryParam(Constants.SCAN_BATCH_SIZE) int batchSize,
@DefaultValue("0") @QueryParam(Constants.SCAN_START_TIME) long startTime,
@DefaultValue(Long.MAX_VALUE + "") @QueryParam(Constants.SCAN_END_TIME) long endTime,
- @DefaultValue("true") @QueryParam(Constants.SCAN_BATCH_SIZE) boolean cacheBlocks) {
+ @DefaultValue("true") @QueryParam(Constants.SCAN_BATCH_SIZE) boolean cacheBlocks,
+ @DefaultValue("") @QueryParam(Constants.SCAN_FILTER) String filters) {
try {
Filter filter = null;
if (scanSpec.indexOf('*') > 0) {
@@ -164,7 +167,20 @@ public class TableResource extends ResourceBase {
tableScan.addFamily(Bytes.toBytes(familysplit[0]));
}
}
- if (filter != null) {
+ FilterList filterList = null;
+ if (StringUtils.isNotEmpty(filters)) {
+ ParseFilter pf = new ParseFilter();
+ Filter filterParam = pf.parseFilterString(filters);
+ if (filter != null) {
+ filterList = new FilterList(filter, filterParam);
+ }
+ else {
+ filter = filterParam;
+ }
+ }
+ if (filterList != null) {
+ tableScan.setFilter(filterList);
+ } else if (filter != null) {
tableScan.setFilter(filter);
}
int fetchSize = this.servlet.getConfiguration().getInt(Constants.SCAN_FETCH_SIZE, 10);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b5db1432/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
index 2e55181..8d09edb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
@@ -27,6 +27,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
+import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
@@ -48,6 +49,9 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.ParseFilter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.rest.client.Client;
import org.apache.hadoop.hbase.rest.client.Cluster;
import org.apache.hadoop.hbase.rest.client.Response;
@@ -87,6 +91,7 @@ public class TestTableScan {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = TEST_UTIL.getConfiguration();
+ conf.set(Constants.CUSTOM_FILTERS, "CustomFilter:" + CustomFilter.class.getName());
TEST_UTIL.startMiniCluster();
REST_TEST_UTIL.startServletContainer(conf);
client = new Client(new Cluster().add("localhost",
@@ -184,7 +189,6 @@ public class TestTableScan {
count = TestScannerResource.countCellSet(model);
assertEquals(15, count);
checkRowsNotNull(model);
-
}
@Test
@@ -459,6 +463,108 @@ public class TestTableScan {
int count = TestScannerResource.countCellSet(model);
assertEquals(0, count);
}
+
+ @Test
+ public void testSimpleFilter() throws IOException, JAXBException {
+ StringBuilder builder = new StringBuilder();
+ builder = new StringBuilder();
+ builder.append("/*");
+ builder.append("?");
+ builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
+ builder.append("&");
+ builder.append(Constants.SCAN_START_ROW + "=aaa");
+ builder.append("&");
+ builder.append(Constants.SCAN_END_ROW + "=aay");
+ builder.append("&");
+ builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8"));
+ Response response =
+ client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
+ Unmarshaller ush = ctx.createUnmarshaller();
+ CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
+ int count = TestScannerResource.countCellSet(model);
+ assertEquals(1, count);
+ assertEquals("aab", new String(model.getRows().get(0).getCells().get(0).getValue()));
+ }
+
+ @Test
+ public void testCompoundFilter() throws IOException, JAXBException {
+ StringBuilder builder = new StringBuilder();
+ builder = new StringBuilder();
+ builder.append("/*");
+ builder.append("?");
+ builder.append(Constants.SCAN_FILTER + "="
+ + URLEncoder.encode("PrefixFilter('abc') AND QualifierFilter(=,'binary:1')", "UTF-8"));
+ Response response =
+ client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
+ Unmarshaller ush = ctx.createUnmarshaller();
+ CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
+ int count = TestScannerResource.countCellSet(model);
+ assertEquals(1, count);
+ assertEquals("abc", new String(model.getRows().get(0).getCells().get(0).getValue()));
+ }
+
+ @Test
+ public void testCustomFilter() throws IOException, JAXBException {
+ StringBuilder builder = new StringBuilder();
+ builder = new StringBuilder();
+ builder.append("/a*");
+ builder.append("?");
+ builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
+ builder.append("&");
+ builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
+ Response response =
+ client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
+ Unmarshaller ush = ctx.createUnmarshaller();
+ CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
+ int count = TestScannerResource.countCellSet(model);
+ assertEquals(1, count);
+ assertEquals("abc", new String(model.getRows().get(0).getCells().get(0).getValue()));
+ }
+
+ @Test
+ public void testNegativeCustomFilter() throws IOException, JAXBException {
+ StringBuilder builder = new StringBuilder();
+ builder = new StringBuilder();
+ builder.append("/b*");
+ builder.append("?");
+ builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
+ builder.append("&");
+ builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
+ Response response =
+ client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
+ Unmarshaller ush = ctx.createUnmarshaller();
+ CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
+ int count = TestScannerResource.countCellSet(model);
+ // Should return no rows as the filters conflict
+ assertEquals(0, count);
+ }
+
+ public static class CustomFilter extends PrefixFilter {
+ private byte[] key = null;
+
+ public CustomFilter(byte[] key) {
+ super(key);
+ }
+
+ @Override
+ public boolean filterRowKey(byte[] buffer, int offset, int length) {
+ int cmp = Bytes.compareTo(buffer, offset, length, this.key, 0, this.key.length);
+ return cmp != 0;
+ }
+
+ public static Filter createFilterFromArguments(ArrayList<byte[]> filterArguments) {
+ byte[] prefix = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0));
+ return new CustomFilter(prefix);
+ }
+ }
/**
* The Class ClientSideCellSetModel which mimics cell set model, and contains listener to perform