You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2024/11/28 05:38:09 UTC
(hbase) 01/02: HBASE-28627 REST ScannerModel doesn't support includeStartRow/includeStopRow (#6374)
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 48df605dbadd4ae5eef37a793025c6cbee89e4b9
Author: chandrasekhar-188k <15...@users.noreply.github.com>
AuthorDate: Wed Nov 27 15:20:27 2024 +0530
HBASE-28627 REST ScannerModel doesn't support includeStartRow/includeStopRow (#6374)
Signed-off-by: Istvan Toth <st...@apache.org>
(cherry picked from commit f9ec1dcbe5df101091dc509e095d2b13f0de0a93)
---
.../org/apache/hadoop/hbase/rest/Constants.java | 3 +
.../apache/hadoop/hbase/rest/ScannerResource.java | 3 +-
.../hadoop/hbase/rest/ScannerResultGenerator.java | 10 +-
.../apache/hadoop/hbase/rest/TableResource.java | 10 +-
.../hadoop/hbase/rest/model/ScannerModel.java | 32 +++++
hbase-rest/src/main/protobuf/ScannerMessage.proto | 2 +
.../hadoop/hbase/rest/TestScannerResource.java | 151 +++++++++++++++++++++
.../hadoop/hbase/rest/client/TestRemoteTable.java | 78 +++++++++++
.../hadoop/hbase/rest/model/TestScannerModel.java | 3 +-
9 files changed, 282 insertions(+), 10 deletions(-)
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
index 8d606793be6..b3d603c660e 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
@@ -112,4 +112,7 @@ public interface Constants {
/** Configuration parameter to set rest client socket timeout */
String REST_CLIENT_SOCKET_TIMEOUT = "hbase.rest.client.socket.timeout";
int DEFAULT_REST_CLIENT_SOCKET_TIMEOUT = 30 * 1000;
+
+ String SCAN_INCLUDE_START_ROW = "includeStartRow";
+ String SCAN_INCLUDE_STOP_ROW = "includeStopRow";
}
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
index 3c652ec6808..2a3da6c3af4 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResource.java
@@ -111,7 +111,8 @@ public class ScannerResource extends ResourceBase {
Filter filter = ScannerResultGenerator.buildFilterFromModel(model);
String tableName = tableResource.getName();
ScannerResultGenerator gen = new ScannerResultGenerator(tableName, spec, filter,
- model.getCaching(), model.getCacheBlocks(), model.getLimit());
+ model.getCaching(), model.getCacheBlocks(), model.getLimit(), model.isIncludeStartRow(),
+ model.isIncludeStopRow());
String id = gen.getID();
ScannerInstanceResource instance =
new ScannerInstanceResource(tableName, id, gen, model.getBatch());
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
index 34627e79e77..99b54ea2693 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ScannerResultGenerator.java
@@ -63,18 +63,20 @@ public class ScannerResultGenerator extends ResultGenerator {
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
final int caching, final boolean cacheBlocks) throws IllegalArgumentException, IOException {
- this(tableName, rowspec, filter, caching, cacheBlocks, -1);
+ this(tableName, rowspec, filter, caching, cacheBlocks, -1, true, false);
}
public ScannerResultGenerator(final String tableName, final RowSpec rowspec, final Filter filter,
- final int caching, final boolean cacheBlocks, int limit) throws IOException {
+ final int caching, final boolean cacheBlocks, int limit, boolean includeStartRow,
+ boolean includeStopRow) throws IOException {
Table table = RESTServlet.getInstance().getTable(tableName);
try {
Scan scan;
if (rowspec.hasEndRow()) {
- scan = new Scan().withStartRow(rowspec.getStartRow()).withStopRow(rowspec.getEndRow());
+ scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow)
+ .withStopRow(rowspec.getEndRow(), includeStopRow);
} else {
- scan = new Scan().withStartRow(rowspec.getStartRow());
+ scan = new Scan().withStartRow(rowspec.getStartRow(), includeStartRow);
}
if (rowspec.hasColumns()) {
byte[][] columns = rowspec.getColumns();
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
index b311e7b0eb7..2c56706e06d 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
@@ -135,7 +135,9 @@ public class TableResource extends ResourceBase {
@DefaultValue("true") @QueryParam(Constants.SCAN_CACHE_BLOCKS) boolean cacheBlocks,
@DefaultValue("false") @QueryParam(Constants.SCAN_REVERSED) boolean reversed,
@QueryParam(Constants.FILTER) String paramFilter,
- @QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64) {
+ @QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64,
+ @DefaultValue("true") @QueryParam(Constants.SCAN_INCLUDE_START_ROW) boolean includeStartRow,
+ @DefaultValue("false") @QueryParam(Constants.SCAN_INCLUDE_STOP_ROW) boolean includeStopRow) {
try {
Filter prefixFilter = null;
Scan tableScan = new Scan();
@@ -144,7 +146,7 @@ public class TableResource extends ResourceBase {
byte[] prefixBytes = Bytes.toBytes(prefix);
prefixFilter = new PrefixFilter(Bytes.toBytes(prefix));
if (startRow.isEmpty()) {
- tableScan.withStartRow(prefixBytes);
+ tableScan.withStartRow(prefixBytes, includeStartRow);
}
}
if (LOG.isTraceEnabled()) {
@@ -158,9 +160,9 @@ public class TableResource extends ResourceBase {
tableScan.readVersions(maxVersions);
tableScan.setTimeRange(startTime, endTime);
if (!startRow.isEmpty()) {
- tableScan.withStartRow(Bytes.toBytes(startRow));
+ tableScan.withStartRow(Bytes.toBytes(startRow), includeStartRow);
}
- tableScan.withStopRow(Bytes.toBytes(endRow));
+ tableScan.withStopRow(Bytes.toBytes(endRow), includeStopRow);
for (String col : column) {
byte[][] parts = CellUtil.parseColumn(Bytes.toBytes(col.trim()));
if (parts.length == 1) {
diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
index 89a8e056e4e..d68e36ad6ff 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
@@ -121,6 +121,28 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
private boolean cacheBlocks = true;
private int limit = -1;
+ private boolean includeStartRow = true;
+
+ private boolean includeStopRow = false;
+
+ @XmlAttribute
+ public boolean isIncludeStopRow() {
+ return includeStopRow;
+ }
+
+ public void setIncludeStopRow(boolean includeStopRow) {
+ this.includeStopRow = includeStopRow;
+ }
+
+ @XmlAttribute
+ public boolean isIncludeStartRow() {
+ return includeStartRow;
+ }
+
+ public void setIncludeStartRow(boolean includeStartRow) {
+ this.includeStartRow = includeStartRow;
+ }
+
/**
* Implement lazily-instantiated singleton as per recipe here:
* http://literatejava.com/jvm/fastest-threadsafe-singleton-jvm/
@@ -730,6 +752,8 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
model.addLabel(label);
}
}
+ model.setIncludeStartRow(scan.includeStartRow());
+ model.setIncludeStopRow(scan.includeStopRow());
return model;
}
@@ -997,6 +1021,8 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
builder.addLabels(label);
}
builder.setCacheBlocks(cacheBlocks);
+ builder.setIncludeStartRow(includeStartRow);
+ builder.setIncludeStopRow(includeStopRow);
return builder.build();
}
@@ -1043,6 +1069,12 @@ public class ScannerModel implements ProtobufMessageHandler, Serializable {
if (builder.hasCacheBlocks()) {
this.cacheBlocks = builder.getCacheBlocks();
}
+ if (builder.hasIncludeStartRow()) {
+ this.includeStartRow = builder.getIncludeStartRow();
+ }
+ if (builder.hasIncludeStopRow()) {
+ this.includeStopRow = builder.getIncludeStopRow();
+ }
return this;
}
diff --git a/hbase-rest/src/main/protobuf/ScannerMessage.proto b/hbase-rest/src/main/protobuf/ScannerMessage.proto
index 4ea8566abb5..71db9f05f21 100644
--- a/hbase-rest/src/main/protobuf/ScannerMessage.proto
+++ b/hbase-rest/src/main/protobuf/ScannerMessage.proto
@@ -31,4 +31,6 @@ message Scanner {
repeated string labels = 10;
optional bool cacheBlocks = 11; // server side block caching hint
optional int32 limit = 12;
+ optional bool includeStartRow = 13;
+ optional bool includeStopRow = 14;
}
diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
index 46be9a14960..ebe49d53da1 100644
--- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
+++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java
@@ -399,4 +399,155 @@ public class TestScannerResource {
Response response = client.delete("/" + TABLE + "/scanner/NONEXISTENT_SCAN");
assertEquals(404, response.getCode());
}
+
+ @Test
+ public void testScannerWithIncludeStartStopRowXML() throws IOException, JAXBException {
+ final int BATCH_SIZE = 5;
+ // new scanner
+ ScannerModel model = new ScannerModel();
+ // model.setBatch(BATCH_SIZE);
+ model.addColumn(Bytes.toBytes(COLUMN_1));
+ model.setStartRow(Bytes.toBytes("aaa"));
+ model.setEndRow(Bytes.toBytes("aae"));
+ StringWriter writer = new StringWriter();
+ marshaller.marshal(model, writer);
+ byte[] body = Bytes.toBytes(writer.toString());
+
+ conf.set("hbase.rest.readonly", "false");
+ Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
+ assertEquals(201, response.getCode());
+ String scannerURI = response.getLocation();
+ assertNotNull(scannerURI);
+
+ // get a cell set
+ response = client.get(scannerURI, Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
+ CellSetModel cellSet =
+ (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
+
+ assertEquals(4, countCellSet(cellSet));
+
+ // test with include the start row false
+ model.setIncludeStartRow(false);
+ writer = new StringWriter();
+ marshaller.marshal(model, writer);
+ body = Bytes.toBytes(writer.toString());
+
+ conf.set("hbase.rest.readonly", "false");
+ response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
+ assertEquals(201, response.getCode());
+ scannerURI = response.getLocation();
+ assertNotNull(scannerURI);
+
+ // get a cell set
+ response = client.get(scannerURI, Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
+ cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
+
+ assertEquals(3, countCellSet(cellSet));
+
+ // test with include stop row true and start row false
+ model.setIncludeStartRow(false);
+ model.setIncludeStopRow(true);
+ writer = new StringWriter();
+ marshaller.marshal(model, writer);
+ body = Bytes.toBytes(writer.toString());
+
+ conf.set("hbase.rest.readonly", "false");
+ response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
+ assertEquals(201, response.getCode());
+ scannerURI = response.getLocation();
+ assertNotNull(scannerURI);
+
+ // get a cell set
+ response = client.get(scannerURI, Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
+ cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
+
+ assertEquals(4, countCellSet(cellSet));
+
+ // test with including the start row true and stop row true
+ model.setIncludeStartRow(true);
+ model.setIncludeStopRow(true);
+ writer = new StringWriter();
+ marshaller.marshal(model, writer);
+ body = Bytes.toBytes(writer.toString());
+
+ conf.set("hbase.rest.readonly", "false");
+ response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML, body);
+ assertEquals(201, response.getCode());
+ scannerURI = response.getLocation();
+ assertNotNull(scannerURI);
+
+ // get a cell set
+ response = client.get(scannerURI, Constants.MIMETYPE_XML);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
+ cellSet = (CellSetModel) unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
+
+ assertEquals(5, countCellSet(cellSet));
+ }
+
+ @Test
+ public void testScannerWithIncludeStartStopRowPB() throws IOException {
+ final int BATCH_SIZE = 10;
+ // new scanner
+ ScannerModel model = new ScannerModel();
+ // model.setBatch(BATCH_SIZE);
+ model.addColumn(Bytes.toBytes(COLUMN_1));
+ model.setStartRow(Bytes.toBytes("aaa"));
+ model.setEndRow(Bytes.toBytes("aae"));
+
+ // test put operation is forbidden in read-only mode
+ conf.set("hbase.rest.readonly", "false");
+ Response response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
+ model.createProtobufOutput());
+ assertEquals(201, response.getCode());
+ String scannerURI = response.getLocation();
+ assertNotNull(scannerURI);
+
+ // get a cell set
+ response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
+ CellSetModel cellSet = new CellSetModel();
+ cellSet.getObjectFromMessage(response.getBody());
+ assertEquals(4, countCellSet(cellSet));
+
+ // test with include start row false
+ model.setIncludeStartRow(false);
+ response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
+ model.createProtobufOutput());
+ assertEquals(201, response.getCode());
+ scannerURI = response.getLocation();
+ assertNotNull(scannerURI);
+
+ // get a cell set
+ response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
+ cellSet = new CellSetModel();
+ cellSet.getObjectFromMessage(response.getBody());
+ assertEquals(3, countCellSet(cellSet));
+
+ // test with include stop row true
+ model.setIncludeStartRow(true);
+ model.setIncludeStopRow(true);
+ response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_PROTOBUF,
+ model.createProtobufOutput());
+ assertEquals(201, response.getCode());
+ scannerURI = response.getLocation();
+ assertNotNull(scannerURI);
+
+ // get a cell set
+ response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
+ assertEquals(200, response.getCode());
+ assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
+ cellSet = new CellSetModel();
+ cellSet.getObjectFromMessage(response.getBody());
+ assertEquals(5, countCellSet(cellSet));
+ }
}
diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
index 914ed5740a9..fcbe7c7ca02 100644
--- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
+++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java
@@ -670,4 +670,82 @@ public class TestRemoteTable {
Thread.sleep(trialPause);
}
}
+
+ @Test
+ public void testScanWithInlcudeStartStopRow() throws Exception {
+ int numTrials = 6;
+
+ // Truncate the test table for inserting test scenarios rows keys
+ TEST_UTIL.getAdmin().disableTable(TABLE);
+ TEST_UTIL.getAdmin().truncateTable(TABLE, false);
+ String row = "testrow";
+
+ try (Table table = TEST_UTIL.getConnection().getTable(TABLE)) {
+ List<Put> puts = new ArrayList<>();
+ Put put = null;
+ for (int i = 1; i <= numTrials; i++) {
+ put = new Put(Bytes.toBytes(row + i));
+ put.addColumn(COLUMN_1, QUALIFIER_1, TS_2, Bytes.toBytes("testvalue" + i));
+ puts.add(put);
+ }
+ table.put(puts);
+ }
+
+ remoteTable =
+ new RemoteHTable(new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())),
+ TEST_UTIL.getConfiguration(), TABLE.toBytes());
+
+ Scan scan =
+ new Scan().withStartRow(Bytes.toBytes(row + "1")).withStopRow(Bytes.toBytes(row + "5"));
+
+ ResultScanner scanner = remoteTable.getScanner(scan);
+ Iterator<Result> resultIterator = scanner.iterator();
+ int counter = 0;
+ while (resultIterator.hasNext()) {
+ byte[] row1 = resultIterator.next().getRow();
+ System.out.println(Bytes.toString(row1));
+ counter++;
+ }
+ assertEquals(4, counter);
+
+ // test with include start row false
+ scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
+ .withStopRow(Bytes.toBytes(row + "5"));
+ scanner = remoteTable.getScanner(scan);
+ resultIterator = scanner.iterator();
+ counter = 0;
+ while (resultIterator.hasNext()) {
+ byte[] row1 = resultIterator.next().getRow();
+ System.out.println(Bytes.toString(row1));
+ counter++;
+ }
+ assertEquals(3, counter);
+
+ // test with include start row false and stop row true
+ scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), false)
+ .withStopRow(Bytes.toBytes(row + "5"), true);
+ scanner = remoteTable.getScanner(scan);
+ resultIterator = scanner.iterator();
+ counter = 0;
+ while (resultIterator.hasNext()) {
+ byte[] row1 = resultIterator.next().getRow();
+ System.out.println(Bytes.toString(row1));
+ counter++;
+ }
+ assertEquals(4, counter);
+
+ // test with include start row true and stop row true
+ scan = new Scan().withStartRow(Bytes.toBytes(row + "1"), true)
+ .withStopRow(Bytes.toBytes(row + "5"), true);
+ scanner = remoteTable.getScanner(scan);
+ resultIterator = scanner.iterator();
+ counter = 0;
+ while (resultIterator.hasNext()) {
+ byte[] row1 = resultIterator.next().getRow();
+ System.out.println(Bytes.toString(row1));
+ counter++;
+ }
+ assertEquals(5, counter);
+ }
+
}
diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java
index a2617c812d5..79984af62df 100644
--- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java
+++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestScannerModel.java
@@ -62,7 +62,8 @@ public class TestScannerModel extends TestModelBase<ScannerModel> {
AS_JSON = "{\"batch\":100,\"caching\":1000,\"cacheBlocks\":false,\"endRow\":\"enp5eng=\","
+ "\"endTime\":1245393318192,\"maxVersions\":2147483647,\"startRow\":\"YWJyYWNhZGFicmE=\","
+ "\"startTime\":1245219839331,\"column\":[\"Y29sdW1uMQ==\",\"Y29sdW1uMjpmb28=\"],"
- + "\"labels\":[\"private\",\"public\"]," + "\"limit\":10000}";
+ + "\"labels\":[\"private\",\"public\"]," + "\"limit\":10000,"
+ + "\"includeStartRow\":true,\"includeStopRow\":false}";
AS_PB = "CgthYnJhY2FkYWJyYRIFenp5engaB2NvbHVtbjEaC2NvbHVtbjI6Zm9vIGQo47qL554kMLDi57mfJDj"
+ "/////B0joB1IHcHJpdmF0ZVIGcHVibGljWABgkE4=";