You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2024/04/16 05:14:24 UTC
(hbase) branch master updated: HBASE-28518 Allow specifying a filter for the REST multiget endpoint
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 4ed17c7117a HBASE-28518 Allow specifying a filter for the REST multiget endpoint
4ed17c7117a is described below
commit 4ed17c7117a4f8f6c114b9edffe2f941b7365c1a
Author: Istvan Toth <st...@apache.org>
AuthorDate: Fri Apr 12 17:29:42 2024 +0200
HBASE-28518 Allow specifying a filter for the REST multiget endpoint
Signed-off-by: Ankit Singhal <an...@apache.org>
---
.../apache/hadoop/hbase/filter/ParseFilter.java | 2 +-
.../org/apache/hadoop/hbase/rest/Constants.java | 3 +-
.../apache/hadoop/hbase/rest/MultiRowResource.java | 31 ++++++++-
.../apache/hadoop/hbase/rest/TableResource.java | 29 ++++++--
.../hadoop/hbase/rest/TestMultiRowResource.java | 81 +++++++++++++++++++---
.../apache/hadoop/hbase/rest/TestTableScan.java | 41 +++++++++--
6 files changed, 161 insertions(+), 26 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java
index c9bd7a1a55c..55133c9d7a1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java
@@ -41,7 +41,7 @@ import org.slf4j.LoggerFactory;
* of this class and a filter object is constructed. This filter object is then wrapped in a scanner
* object which is then returned
* <p>
- * This class addresses the HBASE-4168 JIRA. More documentation on this Filter Language can be found
+ * This class addresses the HBASE-4176 JIRA. More documentation on this Filter Language can be found
* at: https://issues.apache.org/jira/browse/HBASE-4176
*/
@InterfaceAudience.Public
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
index e924be3d7fe..71080de07dd 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
@@ -85,7 +85,8 @@ public interface Constants {
String SCAN_BATCH_SIZE = "batchsize";
String SCAN_LIMIT = "limit";
String SCAN_FETCH_SIZE = "hbase.rest.scan.fetchsize";
- String SCAN_FILTER = "filter";
+ String FILTER = "filter";
+ String FILTER_B64 = "filter_b64";
String SCAN_REVERSED = "reversed";
String SCAN_CACHE_BLOCKS = "cacheblocks";
String CUSTOM_FILTERS = "hbase.rest.custom.filters";
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
index 82900135dc4..47b3c22a7c9 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.hbase.rest;
import java.io.IOException;
+import java.util.Base64;
+import java.util.Base64.Decoder;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.rest.model.CellModel;
import org.apache.hadoop.hbase.rest.model.CellSetModel;
import org.apache.hadoop.hbase.rest.model.RowModel;
@@ -28,9 +32,11 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.javax.ws.rs.Encoded;
import org.apache.hbase.thirdparty.javax.ws.rs.GET;
import org.apache.hbase.thirdparty.javax.ws.rs.HeaderParam;
import org.apache.hbase.thirdparty.javax.ws.rs.Produces;
+import org.apache.hbase.thirdparty.javax.ws.rs.QueryParam;
import org.apache.hbase.thirdparty.javax.ws.rs.core.Context;
import org.apache.hbase.thirdparty.javax.ws.rs.core.MultivaluedMap;
import org.apache.hbase.thirdparty.javax.ws.rs.core.Response;
@@ -40,6 +46,8 @@ import org.apache.hbase.thirdparty.javax.ws.rs.core.UriInfo;
public class MultiRowResource extends ResourceBase implements Constants {
private static final Logger LOG = LoggerFactory.getLogger(MultiRowResource.class);
+ private static final Decoder base64Urldecoder = Base64.getUrlDecoder();
+
TableResource tableResource;
Integer versions = null;
String[] columns = null;
@@ -65,15 +73,34 @@ public class MultiRowResource extends ResourceBase implements Constants {
@GET
@Produces({ MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF })
public Response get(final @Context UriInfo uriInfo,
- final @HeaderParam("Encoding") String keyEncodingHeader) {
+ final @HeaderParam("Encoding") String keyEncodingHeader,
+ @QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64,
+ @QueryParam(Constants.FILTER) String paramFilter) {
MultivaluedMap<String, String> params = uriInfo.getQueryParameters();
String keyEncoding = (keyEncodingHeader != null)
? keyEncodingHeader
: params.getFirst(KEY_ENCODING_QUERY_PARAM_NAME);
servlet.getMetrics().incrementRequests(1);
+
+ byte[] filterBytes = null;
+ if (paramFilterB64 != null) {
+ filterBytes = base64Urldecoder.decode(paramFilterB64);
+ } else if (paramFilter != null) {
+ // Not binary clean
+ filterBytes = paramFilter.getBytes();
+ }
+
try {
+ Filter parsedParamFilter = null;
+ if (filterBytes != null) {
+ // Note that this is a completely different representation of the filters
+ // than the JSON one used in the /table/scanner endpoint
+ ParseFilter pf = new ParseFilter();
+ parsedParamFilter = pf.parseFilterString(filterBytes);
+ }
CellSetModel model = new CellSetModel();
+ // TODO map this to a Table.get(List<Get> gets) call instead of multiple get calls
for (String rk : params.get(ROW_KEYS_PARAM_NAME)) {
RowSpec rowSpec = new RowSpec(rk, keyEncoding);
@@ -88,7 +115,7 @@ public class MultiRowResource extends ResourceBase implements Constants {
}
ResultGenerator generator = ResultGenerator.fromRowSpec(this.tableResource.getName(),
- rowSpec, null, !params.containsKey(NOCACHE_PARAM_NAME));
+ rowSpec, parsedParamFilter, !params.containsKey(NOCACHE_PARAM_NAME));
Cell value = null;
RowModel rowModel = new RowModel(rowSpec.getRow());
if (generator.hasNext()) {
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
index 3a89d79a398..fc3fb6bf705 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hbase.rest;
import java.io.IOException;
+import java.util.Base64.Decoder;
import java.util.List;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Scan;
@@ -46,6 +46,8 @@ public class TableResource extends ResourceBase {
String table;
private static final Logger LOG = LoggerFactory.getLogger(TableResource.class);
+ private static final Decoder base64Urldecoder = java.util.Base64.getUrlDecoder();
+
/**
* Constructor
*/
@@ -103,6 +105,7 @@ public class TableResource extends ResourceBase {
return new RowResource(this, rowspec, versions, check, returnResult, keyEncoding);
}
+ // TODO document
@Path("{suffixglobbingspec: .*\\*/.+}")
public RowResource getRowResourceWithSuffixGlobbing(
// We need the @Encoded decorator so Jersey won't urldecode before
@@ -117,6 +120,8 @@ public class TableResource extends ResourceBase {
return new RowResource(this, suffixglobbingspec, versions, check, returnResult, keyEncoding);
}
+ // TODO document
+ // FIXME handle binary rowkeys (like put and delete does)
@Path("{scanspec: .*[*]$}")
public TableScanResource getScanResource(final @PathParam("scanspec") String scanSpec,
@DefaultValue(Integer.MAX_VALUE + "") @QueryParam(Constants.SCAN_LIMIT) int userRequestedLimit,
@@ -129,7 +134,8 @@ public class TableResource extends ResourceBase {
@DefaultValue(Long.MAX_VALUE + "") @QueryParam(Constants.SCAN_END_TIME) long endTime,
@DefaultValue("true") @QueryParam(Constants.SCAN_CACHE_BLOCKS) boolean cacheBlocks,
@DefaultValue("false") @QueryParam(Constants.SCAN_REVERSED) boolean reversed,
- @DefaultValue("") @QueryParam(Constants.SCAN_FILTER) String paramFilter) {
+ @QueryParam(Constants.FILTER) String paramFilter,
+ @QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64) {
try {
Filter prefixFilter = null;
Scan tableScan = new Scan();
@@ -173,15 +179,24 @@ public class TableResource extends ResourceBase {
}
}
FilterList filterList = new FilterList();
- if (StringUtils.isNotEmpty(paramFilter)) {
+ byte[] filterBytes = null;
+ if (paramFilterB64 != null) {
+ filterBytes = base64Urldecoder.decode(paramFilterB64);
+ } else if (paramFilter != null) {
+ // Not binary clean
+ filterBytes = paramFilter.getBytes();
+ }
+ if (filterBytes != null) {
+ // Note that this is a completely different representation of the filters
+ // than the JSON one used in the /table/scanner endpoint
ParseFilter pf = new ParseFilter();
- Filter parsedParamFilter = pf.parseFilterString(paramFilter);
+ Filter parsedParamFilter = pf.parseFilterString(filterBytes);
if (parsedParamFilter != null) {
filterList.addFilter(parsedParamFilter);
}
- if (prefixFilter != null) {
- filterList.addFilter(prefixFilter);
- }
+ }
+ if (prefixFilter != null) {
+ filterList.addFilter(prefixFilter);
}
if (filterList.size() > 0) {
tableScan.setFilter(filterList);
diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
index 3e7b4e5cf20..e4371dbcc2d 100644
--- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
+++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
@@ -21,11 +21,11 @@ import static org.junit.Assert.assertEquals;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Base64;
+import java.util.Base64.Encoder;
import java.util.Collection;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
@@ -77,10 +77,9 @@ public class TestMultiRowResource {
private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();
+ private static final Encoder base64UrlEncoder = java.util.Base64.getUrlEncoder();
+
private static Client client;
- private static JAXBContext context;
- private static Marshaller marshaller;
- private static Unmarshaller unmarshaller;
private static Configuration conf;
private static Header extraHdr = null;
@@ -105,9 +104,6 @@ public class TestMultiRowResource {
extraHdr = new BasicHeader(RESTServer.REST_CSRF_CUSTOM_HEADER_DEFAULT, "");
TEST_UTIL.startMiniCluster();
REST_TEST_UTIL.startServletContainer(conf);
- context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class);
- marshaller = context.createMarshaller();
- unmarshaller = context.createUnmarshaller();
client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
Admin admin = TEST_UTIL.getAdmin();
if (admin.tableExists(TABLE)) {
@@ -340,4 +336,71 @@ public class TestMultiRowResource {
client.delete(row_5_url, extraHdr);
client.delete(row_6_url, extraHdr);
}
+
+ @Test
+ public void testMultiCellGetFilterJSON() throws IOException {
+ String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1;
+ String row_6_url = "/" + TABLE + "/" + ROW_2 + "/" + COLUMN_2;
+
+ StringBuilder path = new StringBuilder();
+ path.append("/");
+ path.append(TABLE);
+ path.append("/multiget/?row=");
+ path.append(ROW_1);
+ path.append("&row=");
+ path.append(ROW_2);
+
+ if (csrfEnabled) {
+ Response response = client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1));
+ assertEquals(400, response.getCode());
+ }
+
+ client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1), extraHdr);
+ client.post(row_6_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_2), extraHdr);
+
+ Response response = client.get(path.toString(), Constants.MIMETYPE_JSON);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
+
+ // If the filter is used, then we get the same result
+ String positivePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
+ .encodeToString("PrefixFilter('testrow')".getBytes(StandardCharsets.UTF_8.toString())));
+ response = client.get(positivePath, Constants.MIMETYPE_JSON);
+ checkMultiCellGetJSON(response);
+
+ // Same with non binary clean param
+ positivePath = path.toString() + ("&" + Constants.FILTER + "="
+ + URLEncoder.encode("PrefixFilter('testrow')", StandardCharsets.UTF_8.name()));
+ response = client.get(positivePath, Constants.MIMETYPE_JSON);
+ checkMultiCellGetJSON(response);
+
+ // This filter doesn't match the found rows
+ String negativePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
+ .encodeToString("PrefixFilter('notfound')".getBytes(StandardCharsets.UTF_8.toString())));
+ response = client.get(negativePath, Constants.MIMETYPE_JSON);
+ assertEquals(404, response.getCode());
+
+ // Same with non binary clean param
+ negativePath = path.toString() + ("&" + Constants.FILTER + "="
+ + URLEncoder.encode("PrefixFilter('notfound')", StandardCharsets.UTF_8.name()));
+ response = client.get(negativePath, Constants.MIMETYPE_JSON);
+ assertEquals(404, response.getCode());
+
+ // Check with binary parameters
+ // positive case
+ positivePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
+ .encodeToString(Bytes.toBytesBinary("ColumnRangeFilter ('\\x00', true, '\\xff', true)")));
+ response = client.get(positivePath, Constants.MIMETYPE_JSON);
+ checkMultiCellGetJSON(response);
+
+ // negative case
+ negativePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder
+ .encodeToString(Bytes.toBytesBinary("ColumnRangeFilter ('\\x00', true, '1', false)")));
+ response = client.get(negativePath, Constants.MIMETYPE_JSON);
+ assertEquals(404, response.getCode());
+
+ client.delete(row_5_url, extraHdr);
+ client.delete(row_6_url, extraHdr);
+ }
+
}
diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
index 0a55c07bba9..8f0e83c07e9 100644
--- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
+++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
@@ -34,6 +34,7 @@ import java.io.Serializable;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Base64.Encoder;
import java.util.Collections;
import java.util.List;
import javax.xml.bind.JAXBContext;
@@ -94,6 +95,8 @@ public class TestTableScan {
private static int expectedRows3;
private static Configuration conf;
+ private static final Encoder base64UrlEncoder = java.util.Base64.getUrlEncoder();
+
private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility();
@@ -447,7 +450,33 @@ public class TestTableScan {
builder.append("&");
builder.append(Constants.SCAN_END_ROW + "=aay");
builder.append("&");
- builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8"));
+ builder.append(Constants.FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8"));
+ Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
+ Unmarshaller ush = ctx.createUnmarshaller();
+ CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
+ int count = TestScannerResource.countCellSet(model);
+ assertEquals(1, count);
+ assertEquals("aab",
+ new String(model.getRows().get(0).getCells().get(0).getValue(), StandardCharsets.UTF_8));
+ }
+
+ // This only tests the Base64Url encoded filter definition.
+ // base64 encoded row values are not implemented for this endpoint
+ @Test
+ public void testSimpleFilterBase64() throws IOException, JAXBException {
+ StringBuilder builder = new StringBuilder();
+ builder.append("/*");
+ builder.append("?");
+ builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
+ builder.append("&");
+ builder.append(Constants.SCAN_START_ROW + "=aaa");
+ builder.append("&");
+ builder.append(Constants.SCAN_END_ROW + "=aay");
+ builder.append("&");
+ builder.append(Constants.FILTER_B64 + "=" + base64UrlEncoder
+ .encodeToString("PrefixFilter('aab')".getBytes(StandardCharsets.UTF_8.toString())));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
@@ -464,8 +493,8 @@ public class TestTableScan {
StringBuilder builder = new StringBuilder();
builder.append("/abc*");
builder.append("?");
- builder.append(
- Constants.SCAN_FILTER + "=" + URLEncoder.encode("QualifierFilter(=,'binary:1')", "UTF-8"));
+ builder
+ .append(Constants.FILTER + "=" + URLEncoder.encode("QualifierFilter(=,'binary:1')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
@@ -482,7 +511,7 @@ public class TestTableScan {
StringBuilder builder = new StringBuilder();
builder.append("/*");
builder.append("?");
- builder.append(Constants.SCAN_FILTER + "="
+ builder.append(Constants.FILTER + "="
+ URLEncoder.encode("PrefixFilter('abc') AND QualifierFilter(=,'binary:1')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
@@ -502,7 +531,7 @@ public class TestTableScan {
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
- builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
+ builder.append(Constants.FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
@@ -521,7 +550,7 @@ public class TestTableScan {
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
- builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
+ builder.append(Constants.FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);