You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by ch...@apache.org on 2013/08/12 05:39:21 UTC
git commit: CRUNCH-249: Fix HFileTargetIT failure under hadoop2
Updated Branches:
refs/heads/master fd957a920 -> 3f9841136
CRUNCH-249: Fix HFileTargetIT failure under hadoop2
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/3f984113
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/3f984113
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/3f984113
Branch: refs/heads/master
Commit: 3f98411364cec32a0a8c6681dfaabd43caa4dd60
Parents: fd957a9
Author: Chao Shi <ch...@apache.org>
Authored: Mon Aug 12 11:35:23 2013 +0800
Committer: Chao Shi <ch...@apache.org>
Committed: Mon Aug 12 11:35:23 2013 +0800
----------------------------------------------------------------------
.../apache/crunch/io/hbase/HFileTargetIT.java | 36 ++++++++++++++++++++
pom.xml | 4 +--
2 files changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/3f984113/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
index 86d965b..8c1b3f4 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
@@ -53,6 +53,8 @@ import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -63,11 +65,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
+import java.lang.reflect.Field;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
public class HFileTargetIT implements Serializable {
@@ -96,6 +100,38 @@ public class HFileTargetIT implements Serializable {
splits[i] = new byte[] { b };
}
admin.createTable(htable, splits);
+
+ // Set classpath for yarn, otherwise it won't be able to find MRAppMaster
+ // (see CRUNCH-249 and HBASE-8528).
+ HBASE_TEST_UTILITY.getConfiguration().setBoolean("yarn.is.minicluster", true);
+ dirtyFixForJobHistoryServerAddress();
+ }
+
+ /**
+ * We need to set the address of JobHistory server, as it randomly picks a unused port
+ * to listen. Unfortunately, HBaseTestingUtility neither does that nor provides a way
+ * for us to know the picked address. We have to access it using reflection.
+ *
+ * This is necessary when testing with MRv2, but does no harm to MRv1.
+ */
+ private static void dirtyFixForJobHistoryServerAddress() {
+ try {
+ // Retrieve HBASE_TEST_UTILITY.mrCluster via reflection, as it is private.
+ Field mrClusterField = HBaseTestingUtility.class.getDeclaredField("mrCluster");
+ mrClusterField.setAccessible(true);
+ MiniMRCluster mrCluster = (MiniMRCluster) mrClusterField.get(HBASE_TEST_UTILITY);
+ JobConf jobConf = mrCluster.createJobConf();
+ Configuration conf = HBASE_TEST_UTILITY.getConfiguration();
+ String proprety = "mapreduce.jobhistory.address";
+ String value = jobConf.get(proprety);
+ if (value != null) { // maybe null if we're running MRv1
+ conf.set(proprety, value);
+ }
+ } catch (IllegalAccessException e) {
+ throw new AssertionError(e);
+ } catch (NoSuchFieldException e) {
+ throw new AssertionError(e);
+ }
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/crunch/blob/3f984113/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 369041f..e972ce2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -433,10 +433,10 @@ under the License.
</activation>
<properties>
<hadoop.version>2.0.4-alpha</hadoop.version>
- <!-- NOTE: You must build HBase 0.94.1 from source using:
+ <!-- NOTE: You must build HBase 0.94.3 from source using:
mvn clean install -Dhadoop.profile=2.0
for this to work. -->
- <hbase.version>0.94.1</hbase.version>
+ <hbase.version>0.94.3</hbase.version>
<commons-lang.version>2.5</commons-lang.version>
<slf4j.version>1.6.1</slf4j.version>
</properties>