svn commit: r1181553 - /hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java

[ns...@apache.org]

Author: nspiegelberg
Date: Tue Oct 11 02:19:15 2011
New Revision: 1181553

URL: http://svn.apache.org/viewvc?rev=1181553&view=rev
Log:
Ability to Discard Bad HTable Puts

Summary:
While debugging an application consistency issue, we noticed
that a single, synchronous Put request threw a
NoServerForRegionException but eventually succeeded 90 seconds later.
The problem is that failed put requests are not actually removed from
the HTable's writeBuffer. This makes sense for asynchronous puts using
setAutoFlush(false) but don't make sense for the default case where we
expect synchronous operation. We should discard all failed puts for the
synchronous case and provide an API so asynchronous requests can have
their failed puts cleared.

Test Plan:
- test with appserver

Reviewed By: kannan
Reviewers: kannan, jgray
CC: , kannan, nspiegelberg, hbase@lists, pkhemani
Differential Revision: 264990

Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1181553&r1=1181552&r2=1181553&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/client/HTable.java Tue Oct 11 02:19:15 2011
@@ -72,6 +72,7 @@ public class HTable implements HTableInt
   private volatile Configuration configuration;
   private final ArrayList<Put> writeBuffer = new ArrayList<Put>();
   private long writeBufferSize;
+  private boolean clearBufferOnFail;
   private boolean autoFlush;
   private long currentWriteBufferSize;
   protected int scannerCaching;
@@ -135,6 +136,7 @@ public class HTable implements HTableInt
     this.configuration = conf;
     this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
     this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
+    this.clearBufferOnFail = true;
     this.autoFlush = true;
     this.currentWriteBufferSize = 0;
     this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1);
@@ -672,10 +674,15 @@ public class HTable implements HTableInt
       connection.processBatchOfPuts(writeBuffer,
           tableName, pool);
     } finally {
-      // the write buffer was adjusted by processBatchOfPuts
-      currentWriteBufferSize = 0;
-      for (Put aPut : writeBuffer) {
-        currentWriteBufferSize += aPut.heapSize();
+      if (clearBufferOnFail) {
+        writeBuffer.clear();
+        currentWriteBufferSize = 0;
+      } else {
+        // the write buffer was adjusted by processBatchOfPuts
+        currentWriteBufferSize = 0;
+        for (Put aPut : writeBuffer) {
+          currentWriteBufferSize += aPut.heapSize();
+        }
       }
     }
   }
@@ -731,25 +738,45 @@ public class HTable implements HTableInt
   }
 
   /**
+   * See {@link #setAutoFlush(boolean, boolean)}
+   *
+   * @param autoFlush
+   *          Whether or not to enable 'auto-flush'.
+   */
+  public void setAutoFlush(boolean autoFlush) {
+    setAutoFlush(autoFlush, autoFlush);
+  }
+
+  /**
    * Turns 'auto-flush' on or off.
    * <p>
    * When enabled (default), {@link Put} operations don't get buffered/delayed
-   * and are immediately executed.  This is slower but safer.
+   * and are immediately executed. Failed operations are not retried. This is
+   * slower but safer.
    * <p>
-   * Turning this off means that multiple {@link Put}s will be accepted before
-   * any RPC is actually sent to do the write operations.  If the application
-   * dies before pending writes get flushed to HBase, data will be lost.
-   * Other side effects may include the fact that the application thinks a
-   * {@link Put} was executed successfully whereas it was in fact only
-   * buffered and the operation may fail when attempting to flush all pending
-   * writes.  In that case though, the code will retry the failed {@link Put}
-   * upon its next attempt to flush the buffer.
+   * Turning off {@link #autoFlush} means that multiple {@link Put}s will be
+   * accepted before any RPC is actually sent to do the write operations. If the
+   * application dies before pending writes get flushed to HBase, data will be
+   * lost.
+   * <p>
+   * When you turn {@link #autoFlush} off, you should also consider the
+   * {@link #clearBufferOnFail} option. By default, asynchronous {@link Put)
+   * requests will be retried on failure until successful. However, this can
+   * pollute the writeBuffer and slow down batching performance. Additionally,
+   * you may want to issue a number of Put requests and call
+   * {@link #flushCommits()} as a barrier. In both use cases, consider setting
+   * clearBufferOnFail to true to erase the buffer after {@link #flushCommits()}
+   * has been called, regardless of success.
    *
-   * @param autoFlush Whether or not to enable 'auto-flush'.
+   * @param autoFlush
+   *          Whether or not to enable 'auto-flush'.
+   * @param clearBufferOnFail
+   *          Whether to keep Put failures in the writeBuffer
    * @see #flushCommits
    */
-  public void setAutoFlush(boolean autoFlush) {
+  public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
     this.autoFlush = autoFlush;
+    this.clearBufferOnFail = autoFlush || clearBufferOnFail;
   }
 
   /**