You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2016/08/04 00:13:07 UTC
[1/3] hbase git commit: HBASE-16271 Fix logging and re-run the test
in IntegrationTestBulkLoad
Repository: hbase
Updated Branches:
refs/heads/branch-1 12549de43 -> dbd10eee3
refs/heads/branch-1.3 42de52860 -> 57f8ad8d9
refs/heads/master 9b3bc5fac -> 39db239d4
HBASE-16271 Fix logging and re-run the test in IntegrationTestBulkLoad
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/39db239d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/39db239d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/39db239d
Branch: refs/heads/master
Commit: 39db239d4ecf20726af71cd6455af2095b63b61f
Parents: 9b3bc5f
Author: Enis Soztutar <en...@apache.org>
Authored: Wed Aug 3 17:02:35 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Wed Aug 3 17:02:35 2016 -0700
----------------------------------------------------------------------
.../mapreduce/IntegrationTestBulkLoad.java | 32 +++++++++++++++-----
1 file changed, 24 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/39db239d/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
index f00127f..1d8be99 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -204,7 +205,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
HTableDescriptor desc = admin.getTableDescriptor(t);
desc.addCoprocessor(SlowMeCoproScanOperations.class.getName());
HBaseTestingUtility.modifyTableSync(admin, desc);
- //sleep for sometime. Hope is that the regions are closed/opened before
+ //sleep for sometime. Hope is that the regions are closed/opened before
//the sleep returns. TODO: do this better
Thread.sleep(30000);
}
@@ -213,7 +214,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
public void testBulkLoad() throws Exception {
runLoad();
installSlowingCoproc();
- runCheck();
+ runCheckWithRetry();
}
public void runLoad() throws Exception {
@@ -289,7 +290,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
Admin admin = conn.getAdmin();
Table table = conn.getTable(getTablename());
RegionLocator regionLocator = conn.getRegionLocator(getTablename())) {
-
+
// Configure the partitioner and other things needed for HFileOutputFormat.
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
@@ -658,16 +659,31 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
}
private static void logError(String msg, Context context) throws IOException {
- HBaseTestingUtility util = new HBaseTestingUtility(context.getConfiguration());
TableName table = getTableName(context.getConfiguration());
LOG.error("Failure in chain verification: " + msg);
- LOG.error("cluster status:\n" + util.getHBaseClusterInterface().getClusterStatus());
- LOG.error("table regions:\n"
- + Joiner.on("\n").join(util.getHBaseAdmin().getTableRegions(table)));
+ try (Connection connection = ConnectionFactory.createConnection(context.getConfiguration());
+ Admin admin = connection.getAdmin()) {
+ LOG.error("cluster status:\n" + admin.getClusterStatus());
+ LOG.error("table regions:\n"
+ + Joiner.on("\n").join(admin.getTableRegions(table)));
+ }
}
}
+ private void runCheckWithRetry() throws IOException, ClassNotFoundException, InterruptedException {
+ try {
+ runCheck();
+ } catch (Throwable t) {
+ LOG.warn("Received " + StringUtils.stringifyException(t));
+ LOG.warn("Running the check MR Job again to see whether an ephemeral problem or not");
+ runCheck();
+ throw t; // we should still fail the test even if second retry succeeds
+ }
+ // everything green
+ }
+
+
/**
* After adding data to the table start a mr job to
* @throws IOException
@@ -761,7 +777,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
runLoad();
} else if (check) {
installSlowingCoproc();
- runCheck();
+ runCheckWithRetry();
} else {
testBulkLoad();
}
[2/3] hbase git commit: HBASE-16271 Fix logging and re-run the test
in IntegrationTestBulkLoad
Posted by en...@apache.org.
HBASE-16271 Fix logging and re-run the test in IntegrationTestBulkLoad
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dbd10eee
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dbd10eee
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dbd10eee
Branch: refs/heads/branch-1
Commit: dbd10eee3e07ca4b7df3d732ea104f961cddea68
Parents: 12549de
Author: Enis Soztutar <en...@apache.org>
Authored: Wed Aug 3 17:02:35 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Wed Aug 3 17:02:41 2016 -0700
----------------------------------------------------------------------
.../mapreduce/IntegrationTestBulkLoad.java | 32 +++++++++++++++-----
1 file changed, 24 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dbd10eee/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
index b6a6bfe..904b2c8 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -205,7 +206,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
HTableDescriptor desc = admin.getTableDescriptor(t);
desc.addCoprocessor(SlowMeCoproScanOperations.class.getName());
HBaseTestingUtility.modifyTableSync(admin, desc);
- //sleep for sometime. Hope is that the regions are closed/opened before
+ //sleep for sometime. Hope is that the regions are closed/opened before
//the sleep returns. TODO: do this better
Thread.sleep(30000);
}
@@ -214,7 +215,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
public void testBulkLoad() throws Exception {
runLoad();
installSlowingCoproc();
- runCheck();
+ runCheckWithRetry();
}
public void runLoad() throws Exception {
@@ -290,7 +291,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
Admin admin = conn.getAdmin();
Table table = conn.getTable(getTablename());
RegionLocator regionLocator = conn.getRegionLocator(getTablename())) {
-
+
// Configure the partitioner and other things needed for HFileOutputFormat.
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
@@ -659,16 +660,31 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
}
private static void logError(String msg, Context context) throws IOException {
- HBaseTestingUtility util = new HBaseTestingUtility(context.getConfiguration());
TableName table = getTableName(context.getConfiguration());
LOG.error("Failure in chain verification: " + msg);
- LOG.error("cluster status:\n" + util.getHBaseClusterInterface().getClusterStatus());
- LOG.error("table regions:\n"
- + Joiner.on("\n").join(util.getHBaseAdmin().getTableRegions(table)));
+ try (Connection connection = ConnectionFactory.createConnection(context.getConfiguration());
+ Admin admin = connection.getAdmin()) {
+ LOG.error("cluster status:\n" + admin.getClusterStatus());
+ LOG.error("table regions:\n"
+ + Joiner.on("\n").join(admin.getTableRegions(table)));
+ }
}
}
+ private void runCheckWithRetry() throws IOException, ClassNotFoundException, InterruptedException {
+ try {
+ runCheck();
+ } catch (Throwable t) {
+ LOG.warn("Received " + StringUtils.stringifyException(t));
+ LOG.warn("Running the check MR Job again to see whether an ephemeral problem or not");
+ runCheck();
+ throw t; // we should still fail the test even if second retry succeeds
+ }
+ // everything green
+ }
+
+
/**
* After adding data to the table start a mr job to
* @throws IOException
@@ -762,7 +778,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
runLoad();
} else if (check) {
installSlowingCoproc();
- runCheck();
+ runCheckWithRetry();
} else {
testBulkLoad();
}
[3/3] hbase git commit: HBASE-16271 Fix logging and re-run the test
in IntegrationTestBulkLoad
Posted by en...@apache.org.
HBASE-16271 Fix logging and re-run the test in IntegrationTestBulkLoad
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/57f8ad8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/57f8ad8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/57f8ad8d
Branch: refs/heads/branch-1.3
Commit: 57f8ad8d9da299d8161cddac9441cf9187d8318c
Parents: 42de528
Author: Enis Soztutar <en...@apache.org>
Authored: Wed Aug 3 17:02:35 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Wed Aug 3 17:04:31 2016 -0700
----------------------------------------------------------------------
.../mapreduce/IntegrationTestBulkLoad.java | 32 +++++++++++++++-----
1 file changed, 24 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/57f8ad8d/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
index b6a6bfe..904b2c8 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -205,7 +206,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
HTableDescriptor desc = admin.getTableDescriptor(t);
desc.addCoprocessor(SlowMeCoproScanOperations.class.getName());
HBaseTestingUtility.modifyTableSync(admin, desc);
- //sleep for sometime. Hope is that the regions are closed/opened before
+ //sleep for sometime. Hope is that the regions are closed/opened before
//the sleep returns. TODO: do this better
Thread.sleep(30000);
}
@@ -214,7 +215,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
public void testBulkLoad() throws Exception {
runLoad();
installSlowingCoproc();
- runCheck();
+ runCheckWithRetry();
}
public void runLoad() throws Exception {
@@ -290,7 +291,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
Admin admin = conn.getAdmin();
Table table = conn.getTable(getTablename());
RegionLocator regionLocator = conn.getRegionLocator(getTablename())) {
-
+
// Configure the partitioner and other things needed for HFileOutputFormat.
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
@@ -659,16 +660,31 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
}
private static void logError(String msg, Context context) throws IOException {
- HBaseTestingUtility util = new HBaseTestingUtility(context.getConfiguration());
TableName table = getTableName(context.getConfiguration());
LOG.error("Failure in chain verification: " + msg);
- LOG.error("cluster status:\n" + util.getHBaseClusterInterface().getClusterStatus());
- LOG.error("table regions:\n"
- + Joiner.on("\n").join(util.getHBaseAdmin().getTableRegions(table)));
+ try (Connection connection = ConnectionFactory.createConnection(context.getConfiguration());
+ Admin admin = connection.getAdmin()) {
+ LOG.error("cluster status:\n" + admin.getClusterStatus());
+ LOG.error("table regions:\n"
+ + Joiner.on("\n").join(admin.getTableRegions(table)));
+ }
}
}
+ private void runCheckWithRetry() throws IOException, ClassNotFoundException, InterruptedException {
+ try {
+ runCheck();
+ } catch (Throwable t) {
+ LOG.warn("Received " + StringUtils.stringifyException(t));
+ LOG.warn("Running the check MR Job again to see whether an ephemeral problem or not");
+ runCheck();
+ throw t; // we should still fail the test even if second retry succeeds
+ }
+ // everything green
+ }
+
+
/**
* After adding data to the table start a mr job to
* @throws IOException
@@ -762,7 +778,7 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase {
runLoad();
} else if (check) {
installSlowingCoproc();
- runCheck();
+ runCheckWithRetry();
} else {
testBulkLoad();
}