You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2012/07/05 09:22:26 UTC
svn commit: r1357479 - in
/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase:
HBaseTestingUtility.java mapreduce/MapreduceTestingShim.java
mapreduce/TestImportExport.java
Author: jmhsieh
Date: Thu Jul 5 07:22:25 2012
New Revision: 1357479
URL: http://svn.apache.org/viewvc?rev=1357479&view=rev
Log:
HBASE-6330 TestImportExport has been failing against hadoop 0.23/2.0 profile [part2]
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1357479&r1=1357478&r2=1357479&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Thu Jul 5 07:22:25 2012
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.io.encodi
import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -81,6 +82,7 @@ import org.apache.hadoop.hbase.zookeeper
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
@@ -289,21 +291,6 @@ public class HBaseTestingUtility {
createSubDirAndSystemProperty(
"hadoop.log.dir",
testPath, "hadoop-log-dir");
-
- // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but
- // we want our own value to ensure uniqueness on the same machine
- createSubDirAndSystemProperty(
- "hadoop.tmp.dir",
- testPath, "hadoop-tmp-dir");
-
- // Read and modified in org.apache.hadoop.mapred.MiniMRCluster
- createSubDir(
- "mapred.local.dir",
- testPath, "mapred-local-dir");
-
- createSubDirAndSystemProperty(
- "mapred.working.dir",
- testPath, "mapred-working-dir");
}
private void createSubDir(String propertyName, Path parent, String subDirName){
@@ -441,6 +428,24 @@ public class HBaseTestingUtility {
return this.dfsCluster;
}
+ /** This is used before starting HDFS and map-reduce mini-clusters */
+ private void createDirsAndSetProperties() {
+ setupClusterTestDir();
+ System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
+ createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
+ createDirAndSetProperty("mapred_output", MapreduceTestingShim.getMROutputDirProp());
+ createDirAndSetProperty("mapred_local", "mapred.local.dir");
+ }
+
+ private String createDirAndSetProperty(final String relPath, String property) {
+ String path = clusterTestDir.getPath() + "/" + relPath;
+ System.setProperty(property, path);
+ conf.set(property, path);
+ new File(path).mkdirs();
+ LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
+ return path;
+ }
+
/**
* Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
* or does nothing.
@@ -1268,10 +1273,13 @@ public class HBaseTestingUtility {
conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
mrCluster = new MiniMRCluster(servers,
FileSystem.get(conf).getUri().toString(), 1);
+ JobConf jobConf = mrCluster.createJobConf();
LOG.info("Mini mapreduce cluster started");
+
+ // This fixes TestImportTsv but breaks TestImportExport tests
conf.set("mapred.job.tracker",
mrCluster.createJobConf().get("mapred.job.tracker"));
- /* this for mrv2 support */
+ // this for mrv2 support; mr1 ignores this
conf.set("mapreduce.framework.name", "yarn");
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java?rev=1357479&r1=1357478&r2=1357479&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java Thu Jul 5 07:22:25 2012
@@ -49,11 +49,17 @@ abstract public class MapreduceTestingSh
abstract public JobContext newJobContext(Configuration jobConf)
throws IOException;
+ abstract public String obtainMROutputDirProp();
+
public static JobContext createJobContext(Configuration jobConf)
throws IOException {
return instance.newJobContext(jobConf);
}
+ public static String getMROutputDirProp() {
+ return instance.obtainMROutputDirProp();
+ }
+
private static class MapreduceV1Shim extends MapreduceTestingShim {
public JobContext newJobContext(Configuration jobConf) throws IOException {
// Implementing:
@@ -68,6 +74,11 @@ abstract public class MapreduceTestingSh
"Failed to instantiate new JobContext(jobConf, new JobID())", e);
}
}
+
+ @Override
+ public String obtainMROutputDirProp() {
+ return "mapred.output.dir";
+ }
};
private static class MapreduceV2Shim extends MapreduceTestingShim {
@@ -83,6 +94,12 @@ abstract public class MapreduceTestingSh
"Failed to return from Job.getInstance(jobConf)");
}
}
+
+ @Override
+ public String obtainMROutputDirProp() {
+ // o.a.h.mapreduce.lib.output FileOutputFormat.OUTDIR
+ return "mapreduce.output.fileoutputformat.outputdir";
+ }
};
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java?rev=1357479&r1=1357478&r2=1357479&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java Thu Jul 5 07:22:25 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.mapreduce;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
@@ -27,7 +28,6 @@ import org.apache.hadoop.hbase.HColumnDe
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
import org.junit.After;
@@ -44,7 +45,6 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import static org.junit.Assert.assertEquals;
@Category(MediumTests.class)
public class TestImportExport {
@@ -58,12 +58,11 @@ public class TestImportExport {
private static final byte[] QUAL = Bytes.toBytes("q");
private static final String OUTPUT_DIR = "outputdir";
- private static MiniHBaseCluster cluster;
private static long now = System.currentTimeMillis();
@BeforeClass
public static void beforeClass() throws Exception {
- cluster = UTIL.startMiniCluster();
+ UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
}
@@ -105,16 +104,16 @@ public class TestImportExport {
"1000"
};
- GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+ GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
Configuration conf = opts.getConfiguration();
args = opts.getRemainingArgs();
Job job = Export.createSubmittableJob(conf, args);
job.getConfiguration().set("mapreduce.framework.name", "yarn");
job.waitForCompletion(false);
+ HBaseFsck.debugLsr(conf, new Path("."));
assertTrue(job.isSuccessful());
-
String IMPORT_TABLE = "importTableSimpleCase";
t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
args = new String[] {
@@ -123,13 +122,14 @@ public class TestImportExport {
OUTPUT_DIR
};
- opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+ opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
conf = opts.getConfiguration();
args = opts.getRemainingArgs();
job = Import.createSubmittableJob(conf, args);
job.getConfiguration().set("mapreduce.framework.name", "yarn");
job.waitForCompletion(false);
+ HBaseFsck.debugLsr(conf, new Path("."));
assertTrue(job.isSuccessful());
Get g = new Get(ROW1);
@@ -174,7 +174,7 @@ public class TestImportExport {
"1000"
};
- GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+ GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
Configuration conf = opts.getConfiguration();
args = opts.getRemainingArgs();
@@ -198,7 +198,7 @@ public class TestImportExport {
OUTPUT_DIR
};
- opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
+ opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
conf = opts.getConfiguration();
args = opts.getRemainingArgs();