You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2010/01/04 23:55:32 UTC
svn commit: r895818 - in /hadoop/hbase/branches/0.20: CHANGES.txt
src/java/org/apache/hadoop/hbase/HConstants.java
src/java/org/apache/hadoop/hbase/client/HTable.java
src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Author: stack
Date: Mon Jan 4 22:55:28 2010
New Revision: 895818
URL: http://svn.apache.org/viewvc?rev=895818&view=rev
Log:
HBASE-1996 Configure scanner buffer in bytes instead of number of rows
Modified:
hadoop/hbase/branches/0.20/CHANGES.txt
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HConstants.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/HTable.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=895818&r1=895817&r2=895818&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Mon Jan 4 22:55:28 2010
@@ -82,6 +82,8 @@
HBASE-2083 [EC2] HDFS DataNode no longer required on master
HBASE-2084 [EC2] JAVA_HOME handling broken
HBASE-2053 Upper bound of outstanding WALs can be overrun
+ HBASE-1996 Configure scanner buffer in bytes instead of number of rows
+ (Erik Rozendaal and Dave Latham via Stack)
Release 0.20.2 - November 18th, 2009
INCOMPATIBLE CHANGES
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HConstants.java?rev=895818&r1=895817&r2=895818&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/HConstants.java Mon Jan 4 22:55:28 2010
@@ -284,4 +284,20 @@
TABLE_SET_HTD,
TABLE_SPLIT
}
+
+ /**
+ * Parameter name for maximum number of bytes returned when calling a
+ * scanner's next method.
+ */
+ public static String HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY = "hbase.client.scanner.max.result.size";
+
+ /**
+ * Maximum number of bytes returned when calling a scanner's next method.
+ * Note that when a single row is larger than this limit the row is still
+ * returned completely.
+ *
+ * The default value is unlimited.
+ */
+ public static long DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE = Long.MAX_VALUE;
+
}
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=895818&r1=895817&r2=895818&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/HTable.java Mon Jan 4 22:55:28 2010
@@ -70,7 +70,8 @@
private boolean autoFlush;
private long currentWriteBufferSize;
protected int scannerCaching;
-
+ private long maxScannerResultSize;
+
/**
* Creates an object to access a HBase table
*
@@ -129,6 +130,9 @@
this.autoFlush = true;
this.currentWriteBufferSize = 0;
this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1);
+ this.maxScannerResultSize = conf.getLong(
+ HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
+ HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
}
/**
@@ -1954,6 +1958,7 @@
}
if (cache.size() == 0) {
Result [] values = null;
+ long remainingResultSize = maxScannerResultSize;
int countdown = this.caching;
// We need to reset it if it's a new callable that was created
// with a countdown in nextScanner
@@ -2001,12 +2006,15 @@
if (values != null && values.length > 0) {
for (Result rs : values) {
cache.add(rs);
+ for (KeyValue kv : rs.raw()) {
+ remainingResultSize -= kv.heapSize();
+ }
countdown--;
this.lastResult = rs;
}
}
// Values == null means server-side filter has determined we must STOP
- } while (countdown > 0 && nextScanner(countdown, values == null));
+ } while (remainingResultSize > 0 && countdown > 0 && nextScanner(countdown, values == null));
}
if (cache.size() > 0) {
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=895818&r1=895817&r2=895818&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Mon Jan 4 22:55:28 2010
@@ -160,6 +160,8 @@
private final int serverLeaseTimeout;
protected final int numRegionsToReport;
+
+ private final long maxScannerResultSize;
// Remote HMaster
private HMasterRegionInterface hbaseMaster;
@@ -262,6 +264,10 @@
sleeper = new Sleeper(this.msgInterval, this.stopRequested);
+ this.maxScannerResultSize = conf.getLong(
+ HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
+ HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
+
// Task thread to process requests from Master
this.worker = new Worker();
@@ -1878,12 +1884,16 @@
}
this.leases.renewLease(scannerName);
List<Result> results = new ArrayList<Result>();
- for (int i = 0; i < nbRows; i++) {
+ long currentScanResultSize = 0;
+ for (int i = 0; i < nbRows && currentScanResultSize < maxScannerResultSize; i++) {
requestCount.incrementAndGet();
// Collect values to be returned here
List<KeyValue> values = new ArrayList<KeyValue>();
boolean moreRows = s.next(values);
if (!values.isEmpty()) {
+ for (KeyValue kv : values) {
+ currentScanResultSize += kv.heapSize();
+ }
results.add(new Result(values));
}
if (!moreRows) {