You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/05/02 21:08:17 UTC
svn commit: r1333157 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/coprocessor/
Author: tedyu
Date: Wed May 2 19:08:16 2012
New Revision: 1333157
URL: http://svn.apache.org/viewvc?rev=1333157&view=rev
Log:
HBASE-2214 Do HBASE-1996 -- setting size to return in scan rather than count of rows -- properly (Ferdy Galema)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/Scan.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java?rev=1333157&r1=1333156&r2=1333157&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java Wed May 2 19:08:16 2012
@@ -95,9 +95,13 @@ public class ClientScanner extends Abstr
this.tableName = tableName;
this.lastNext = System.currentTimeMillis();
this.connection = connection;
- this.maxScannerResultSize = conf.getLong(
+ if (scan.getMaxResultSize() > 0) {
+ this.maxScannerResultSize = scan.getMaxResultSize();
+ } else {
+ this.maxScannerResultSize = conf.getLong(
HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
+ }
this.scannerTimeout = (int) conf.getLong(
HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/Scan.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/Scan.java?rev=1333157&r1=1333156&r2=1333157&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/Scan.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/Scan.java Wed May 2 19:08:16 2012
@@ -51,8 +51,11 @@ import java.util.TreeSet;
* To scan everything for each row, instantiate a Scan object.
* <p>
* To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}.
- * If caching is NOT set, we will use the caching value of the hosting
- * {@link HTable}. See {@link HTable#setScannerCaching(int)}.
+ * If caching is NOT set, we will use the caching value of the hosting {@link HTable}. See
+ * {@link HTable#setScannerCaching(int)}. In addition to row caching, it is possible to specify a
+ * maximum result size, using {@link #setMaxResultSize(long)}. When both are used,
+ * single server requests are limited by either number of rows or maximum result size, whichever
+ * limit comes first.
* <p>
* To further define the scope of what to get when scanning, perform additional
* methods as outlined below.
@@ -84,7 +87,7 @@ public class Scan extends OperationWithA
private static final String RAW_ATTR = "_raw_";
private static final String ISOLATION_LEVEL = "_isolationlevel_";
- private static final byte SCAN_VERSION = (byte)2;
+ private static final byte SCAN_VERSION = (byte)3;
private byte [] startRow = HConstants.EMPTY_START_ROW;
private byte [] stopRow = HConstants.EMPTY_END_ROW;
private int maxVersions = 1;
@@ -100,6 +103,7 @@ public class Scan extends OperationWithA
* -1 means no caching
*/
private int caching = -1;
+ private long maxResultSize = -1;
private boolean cacheBlocks = true;
private Filter filter = null;
private TimeRange tr = new TimeRange();
@@ -149,6 +153,7 @@ public class Scan extends OperationWithA
maxVersions = scan.getMaxVersions();
batch = scan.getBatch();
caching = scan.getCaching();
+ maxResultSize = scan.getMaxResultSize();
cacheBlocks = scan.getCacheBlocks();
filter = scan.getFilter(); // clone?
TimeRange ctr = scan.getTimeRange();
@@ -323,6 +328,24 @@ public class Scan extends OperationWithA
}
/**
+ * @return the maximum result size in bytes. See {@link #setMaxResultSize(long)}
+ */
+ public long getMaxResultSize() {
+ return maxResultSize;
+ }
+
+ /**
+ * Set the maximum result size. The default is -1; this means that no specific
+ * maximum result size will be set for this scan, and the global configured
+ * value will be used instead. (Defaults to unlimited).
+ *
+ * @param maxResultSize The maximum result size in bytes.
+ */
+ public void setMaxResultSize(long maxResultSize) {
+ this.maxResultSize = maxResultSize;
+ }
+
+ /**
* Apply the specified server-side filter when performing the Scan.
* @param filter filter to run on the server
* @return this
@@ -500,6 +523,7 @@ public class Scan extends OperationWithA
map.put("maxVersions", this.maxVersions);
map.put("batch", this.batch);
map.put("caching", this.caching);
+ map.put("maxResultSize", this.maxResultSize);
map.put("cacheBlocks", this.cacheBlocks);
List<Long> timeRange = new ArrayList<Long>();
timeRange.add(this.tr.getMin());
@@ -582,6 +606,9 @@ public class Scan extends OperationWithA
if (version > 1) {
readAttributes(in);
}
+ if (version > 2) {
+ this.maxResultSize = in.readLong();
+ }
}
public void write(final DataOutput out)
@@ -615,6 +642,7 @@ public class Scan extends OperationWithA
}
}
writeAttributes(out);
+ out.writeLong(maxResultSize);
}
/**
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1333157&r1=1333156&r2=1333157&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed May 2 19:08:16 2012
@@ -3231,6 +3231,7 @@ public class HRegion implements HeapSize
private int isScan;
private boolean filterClosed = false;
private long readPt;
+ private long maxResultSize;
public HRegionInfo getRegionInfo() {
return regionInfo;
@@ -3238,6 +3239,7 @@ public class HRegion implements HeapSize
RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
//DebugPrint.println("HRegionScanner.<init>");
+ this.maxResultSize = scan.getMaxResultSize();
this.filter = scan.getFilter();
this.batch = scan.getBatch();
if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
@@ -3281,6 +3283,11 @@ public class HRegion implements HeapSize
this(scan, null);
}
+ @Override
+ public long getMaxResultSize() {
+ return maxResultSize;
+ }
+
/**
* Reset both the filter and the old filter.
*/
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1333157&r1=1333156&r2=1333157&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Wed May 2 19:08:16 2012
@@ -2385,9 +2385,13 @@ public class HRegionServer implements HR
: results.toArray(new Result[0]);
}
}
-
- for (int i = 0; i < nbRows
- && currentScanResultSize < maxScannerResultSize; i++) {
+ long maxResultSize;
+ if (s.getMaxResultSize() > 0) {
+ maxResultSize = s.getMaxResultSize();
+ } else {
+ maxResultSize = maxScannerResultSize;
+ }
+ for (int i = 0; i < nbRows && currentScanResultSize < maxResultSize; i++) {
requestCount.incrementAndGet();
// Collect values to be returned here
boolean moreRows = s.next(values);
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java?rev=1333157&r1=1333156&r2=1333157&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionScanner.java Wed May 2 19:08:16 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionse
import java.io.IOException;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.client.Scan;
/**
* RegionScanner describes iterators over rows in an HRegion.
@@ -49,4 +50,8 @@ public interface RegionScanner extends I
*/
public boolean reseek(byte[] row) throws IOException;
+ /**
+ * @return The preferred max buffersize. See {@link Scan#setMaxResultSize(long)}
+ */
+ public long getMaxResultSize();
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java?rev=1333157&r1=1333156&r2=1333157&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorInterface.java Wed May 2 19:08:16 2012
@@ -89,6 +89,11 @@ public class TestCoprocessorInterface ex
return false;
}
+ @Override
+ public long getMaxResultSize() {
+ return delegate.getMaxResultSize();
+ }
+
}
public static class CoprocessorImpl extends BaseRegionObserver {