You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2012/05/21 23:02:32 UTC
svn commit: r1341205 - in /hbase/branches/0.92: ./
src/main/java/org/apache/hadoop/hbase/mapred/
src/main/java/org/apache/hadoop/hbase/mapreduce/
src/test/java/org/apache/hadoop/hbase/mapred/
Author: jmhsieh
Date: Mon May 21 21:02:31 2012
New Revision: 1341205
URL: http://svn.apache.org/viewvc?rev=1341205&view=rev
Log:
HBASE-5757 TableInputFormat should handle as many errors as possible (Jan Lukavsky)
Modified:
hbase/branches/0.92/CHANGES.txt
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1341205&r1=1341204&r2=1341205&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Mon May 21 21:02:31 2012
@@ -69,6 +69,7 @@ Release 0.92.2 - Unreleased
HBASE-6054 0.92 failing because of missing commons-io after upgrade to hadoop 1.0.3
HBASE-5920 New Compactions Logic can silently prevent user-initiated compactions from occurring
(Derek Wollenstein)
+ HBASE-5757 TableInputFormat should handle as many errors as possible (Jan Lukavsky)
IMPROVEMENTS
HBASE-5592 Make it easier to get a table from shell (Ben West)
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java?rev=1341205&r1=1341204&r2=1341205&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapred/TableRecordReaderImpl.java Mon May 21 21:02:31 2012
@@ -175,9 +175,9 @@ public class TableRecordReaderImpl {
Result result;
try {
result = this.scanner.next();
- } catch (DoNotRetryIOException e) {
- throw e;
} catch (IOException e) {
+ // try to handle all IOExceptions by restarting
+ // the scanner, if the second call fails, it will be rethrown
LOG.debug("recovered from " + StringUtils.stringifyException(e));
if (lastSuccessfulRow == null) {
LOG.warn("We are restarting the first next() invocation," +
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java?rev=1341205&r1=1341204&r2=1341205&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java Mon May 21 21:02:31 2012
@@ -23,7 +23,6 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -131,9 +130,9 @@ public class TableRecordReaderImpl {
if (value == null) value = new Result();
try {
value = this.scanner.next();
- } catch (DoNotRetryIOException e) {
- throw e;
} catch (IOException e) {
+ // try to handle all IOExceptions by restarting
+ // the scanner, if the second call fails, it will be rethrown
LOG.debug("recovered from " + StringUtils.stringifyException(e));
if (lastSuccessfulRow == null) {
LOG.warn("We are restarting the first next() invocation," +
Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java?rev=1341205&r1=1341204&r2=1341205&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java (original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java Mon May 21 21:02:31 2012
@@ -195,16 +195,16 @@ public class TestTableInputFormat {
*
* @throws IOException
*/
- static HTable createIOEScannerTable(byte[] name) throws IOException {
+ static HTable createIOEScannerTable(byte[] name, final int failCnt)
+ throws IOException {
// build up a mock scanner stuff to fail the first time
Answer<ResultScanner> a = new Answer<ResultScanner>() {
- boolean first = true;
+ int cnt = 0;
@Override
public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
// first invocation return the busted mock scanner
- if (first) {
- first = false;
+ if (cnt++ < failCnt) {
// create mock ResultScanner that always fails.
Scan scan = mock(Scan.class);
doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe
@@ -230,16 +230,16 @@ public class TestTableInputFormat {
*
* @throws IOException
*/
- static HTable createDNRIOEScannerTable(byte[] name) throws IOException {
+ static HTable createDNRIOEScannerTable(byte[] name, final int failCnt)
+ throws IOException {
// build up a mock scanner stuff to fail the first time
Answer<ResultScanner> a = new Answer<ResultScanner>() {
- boolean first = true;
+ int cnt = 0;
@Override
public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
// first invocation return the busted mock scanner
- if (first) {
- first = false;
+ if (cnt++ < failCnt) {
// create mock ResultScanner that always fails.
Scan scan = mock(Scan.class);
doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe
@@ -280,7 +280,18 @@ public class TestTableInputFormat {
*/
@Test
public void testTableRecordReaderScannerFail() throws IOException {
- HTable htable = createIOEScannerTable("table2".getBytes());
+ HTable htable = createIOEScannerTable("table2".getBytes(), 1);
+ runTestMapred(htable);
+ }
+
+ /**
+ * Run test assuming Scanner IOException failure using mapred api,
+ *
+ * @throws IOException
+ */
+ @Test(expected = IOException.class)
+ public void testTableRecordReaderScannerFailTwice() throws IOException {
+ HTable htable = createIOEScannerTable("table3".getBytes(), 2);
runTestMapred(htable);
}
@@ -290,9 +301,21 @@ public class TestTableInputFormat {
*
* @throws DoNotRetryIOException
*/
- @Test(expected = DoNotRetryIOException.class)
+ @Test
public void testTableRecordReaderScannerTimeout() throws IOException {
- HTable htable = createDNRIOEScannerTable("table3".getBytes());
+ HTable htable = createDNRIOEScannerTable("table4".getBytes(), 1);
+ runTestMapred(htable);
+ }
+
+ /**
+ * Run test assuming UnknownScannerException (which is a type of
+ * DoNotRetryIOException) using mapred api.
+ *
+ * @throws DoNotRetryIOException
+ */
+ @Test(expected = DoNotRetryIOException.class)
+ public void testTableRecordReaderScannerTimeoutTwice() throws IOException {
+ HTable htable = createDNRIOEScannerTable("table5".getBytes(), 2);
runTestMapred(htable);
}
@@ -318,7 +341,20 @@ public class TestTableInputFormat {
@Test
public void testTableRecordReaderScannerFailMapreduce() throws IOException,
InterruptedException {
- HTable htable = createIOEScannerTable("table2-mr".getBytes());
+ HTable htable = createIOEScannerTable("table2-mr".getBytes(), 1);
+ runTestMapreduce(htable);
+ }
+
+ /**
+ * Run test assuming Scanner IOException failure using newer mapreduce api
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test(expected = IOException.class)
+ public void testTableRecordReaderScannerFailMapreduceTwice() throws IOException,
+ InterruptedException {
+ HTable htable = createIOEScannerTable("table3-mr".getBytes(), 2);
runTestMapreduce(htable);
}
@@ -329,10 +365,24 @@ public class TestTableInputFormat {
* @throws InterruptedException
* @throws DoNotRetryIOException
*/
- @Test(expected = DoNotRetryIOException.class)
+ @Test
public void testTableRecordReaderScannerTimeoutMapreduce()
throws IOException, InterruptedException {
- HTable htable = createDNRIOEScannerTable("table3-mr".getBytes());
+ HTable htable = createDNRIOEScannerTable("table4-mr".getBytes(), 1);
+ runTestMapreduce(htable);
+ }
+
+ /**
+ * Run test assuming UnknownScannerException (which is a type of
+ * DoNotRetryIOException) using newer mapreduce api
+ *
+ * @throws InterruptedException
+ * @throws DoNotRetryIOException
+ */
+ @Test(expected = DoNotRetryIOException.class)
+ public void testTableRecordReaderScannerTimeoutMapreduceTwice()
+ throws IOException, InterruptedException {
+ HTable htable = createDNRIOEScannerTable("table5-mr".getBytes(), 2);
runTestMapreduce(htable);
}
}