You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2012/12/05 00:08:35 UTC
svn commit: r1417233 - in /hbase/trunk:
hbase-common/src/main/java/org/apache/hadoop/hbase/
hbase-server/src/main/java/org/apache/hadoop/hbase/client/
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/
hbase-server/src/main/resources/ hbase-server...
Author: larsh
Date: Tue Dec 4 23:08:34 2012
New Revision: 1417233
URL: http://svn.apache.org/viewvc?rev=1417233&view=rev
Log:
HBASE-7008 Set scanner caching to a better default, disable Nagles
Modified:
hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
hbase/trunk/hbase-server/src/main/resources/hbase-default.xml
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Dec 4 23:08:34 2012
@@ -549,6 +549,16 @@ public final class HConstants {
public static int DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT = 10;
/**
+ * Parameter name to set the default scanner caching for all clients.
+ */
+ public static String HBASE_CLIENT_SCANNER_CACHING = "hbase.client.scanner.caching";
+
+ /**
+ * Default value for {@link #HBASE_CLIENT_SCANNER_CACHING}
+ */
+ public static int DEFAULT_HBASE_CLIENT_SCANNER_CACHING = 100;
+
+ /**
* Parameter name for number of rows that will be fetched when calling next on
* a scanner if it is not served from memory. Higher caching values will
* enable faster scanners but will eat up more memory and some calls of next
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java Tue Dec 4 23:08:34 2012
@@ -121,7 +121,9 @@ public class ClientScanner extends Abstr
if (this.scan.getCaching() > 0) {
this.caching = this.scan.getCaching();
} else {
- this.caching = conf.getInt("hbase.client.scanner.caching", 1);
+ this.caching = conf.getInt(
+ HConstants.HBASE_CLIENT_SCANNER_CACHING,
+ HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
}
// initialize the scanner
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java Tue Dec 4 23:08:34 2012
@@ -258,7 +258,8 @@ public class HTable implements HTableInt
this.autoFlush = true;
this.currentWriteBufferSize = 0;
this.scannerCaching = this.configuration.getInt(
- "hbase.client.scanner.caching", 1);
+ HConstants.HBASE_CLIENT_SCANNER_CACHING,
+ HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
this.maxKeyValueSize = this.configuration.getInt(
"hbase.client.keyvalue.maxsize", -1);
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseClient.java Tue Dec 4 23:08:34 2012
@@ -1157,7 +1157,7 @@ public class HBaseClient {
conf.getInt("hbase.ipc.client.connection.maxidletime", 10000); //10s
this.maxRetries = conf.getInt("hbase.ipc.client.connect.max.retries", 0);
this.failureSleep = conf.getInt("hbase.client.pause", 1000);
- this.tcpNoDelay = conf.getBoolean("hbase.ipc.client.tcpnodelay", false);
+ this.tcpNoDelay = conf.getBoolean("hbase.ipc.client.tcpnodelay", true);
this.tcpKeepAlive = conf.getBoolean("hbase.ipc.client.tcpkeepalive", true);
this.pingInterval = getPingInterval(conf);
if (LOG.isDebugEnabled()) {
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/HBaseServer.java Tue Dec 4 23:08:34 2012
@@ -1938,7 +1938,7 @@ public abstract class HBaseServer implem
this.port = listener.getAddress().getPort();
this.rpcMetrics = new HBaseRpcMetrics(
serverName, Integer.toString(this.port));
- this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", false);
+ this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", true);
this.tcpKeepAlive = conf.getBoolean("ipc.server.tcpkeepalive", true);
this.warnDelayedCalls = conf.getInt(WARN_DELAYED_CALLS,
Modified: hbase/trunk/hbase-server/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/resources/hbase-default.xml?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/hbase-server/src/main/resources/hbase-default.xml Tue Dec 4 23:08:34 2012
@@ -133,7 +133,7 @@
</property>
<property>
<name>hbase.client.scanner.caching</name>
- <value>1</value>
+ <value>100</value>
<description>Number of rows that will be fetched when calling next
on a scanner if it is not served from (local, client) memory. Higher
caching values will enable faster scanners but will eat up more memory
@@ -528,6 +528,13 @@
used for client / server RPC call marshalling.
</description>
</property>
+ <property>
+ <name>hbase.ipc.client.tcpnodelay</name>
+ <value>true</value>
+ <description>Set no delay on rpc socket connections. See
+ http://docs.oracle.com/javase/1.5.0/docs/api/java/net/Socket.html#getTcpNoDelay()
+ </description>
+ </property>
<!-- The following properties configure authentication information for
HBase processes when using Kerberos security. There are no default
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java Tue Dec 4 23:08:34 2012
@@ -4663,7 +4663,20 @@ public class TestFromClientSide {
// turn on scan metrics
Scan scan = new Scan();
scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+ scan.setCaching(numRecords+1);
ResultScanner scanner = ht.getScanner(scan);
+ for (Result result : scanner.next(numRecords - 1)) {
+ }
+ scanner.close();
+ // need to have at one next roundtrip in order to collect metrics
+ // here we have less than <numRecord>+1 KVs, so no metrics were collected
+ assertNull(scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA));
+
+ // set caching to 1, becasue metrics are collected in each roundtrip only
+ scan = new Scan();
+ scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
+ scan.setCaching(1);
+ scanner = ht.getScanner(scan);
// per HBASE-5717, this should still collect even if you don't run all the way to
// the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
for (Result result : scanner.next(numRecords - 1)) {
@@ -4677,6 +4690,7 @@ public class TestFromClientSide {
// now, test that the metrics are still collected even if you don't call close, but do
// run past the end of all the records
Scan scanWithoutClose = new Scan();
+ scanWithoutClose.setCaching(1);
scanWithoutClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
for (Result result : scannerWithoutClose.next(numRecords + 1)) {
@@ -4688,6 +4702,8 @@ public class TestFromClientSide {
// finally, test that the metrics are collected correctly if you both run past all the records,
// AND close the scanner
Scan scanWithClose = new Scan();
+ // make sure we can set caching up to the number of a scanned values
+ scanWithClose.setCaching(numRecords);
scanWithClose.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
for (Result result : scannerWithClose.next(numRecords + 1)) {
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java?rev=1417233&r1=1417232&r2=1417233&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestScannerTimeout.java Tue Dec 4 23:08:34 2012
@@ -98,6 +98,7 @@ public class TestScannerTimeout {
public void test2481() throws Exception {
LOG.info("START ************ test2481");
Scan scan = new Scan();
+ scan.setCaching(1);
HTable table =
new HTable(new Configuration(TEST_UTIL.getConfiguration()), TABLE_NAME);
ResultScanner r = table.getScanner(scan);