You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2012/07/05 21:58:54 UTC
svn commit: r1357853 - in
/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase:
HBaseTestingUtility.java mapreduce/MapreduceTestingShim.java
mapreduce/TestImportExport.java
Author: jmhsieh
Date: Thu Jul 5 19:58:54 2012
New Revision: 1357853
URL: http://svn.apache.org/viewvc?rev=1357853&view=rev
Log:
Revert HBASE-6330 - hadoop 2.0 was fixed but this broke hadoop 1.0 build
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1357853&r1=1357852&r2=1357853&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Thu Jul 5 19:58:54 2012
@@ -61,7 +61,6 @@ import org.apache.hadoop.hbase.io.encodi
import org.apache.hadoop.hbase.io.hfile.ChecksumUtil;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.mapreduce.MapreduceTestingShim;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -82,7 +81,6 @@ import org.apache.hadoop.hbase.zookeeper
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
@@ -291,6 +289,21 @@ public class HBaseTestingUtility {
createSubDirAndSystemProperty(
"hadoop.log.dir",
testPath, "hadoop-log-dir");
+
+ // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but
+ // we want our own value to ensure uniqueness on the same machine
+ createSubDirAndSystemProperty(
+ "hadoop.tmp.dir",
+ testPath, "hadoop-tmp-dir");
+
+ // Read and modified in org.apache.hadoop.mapred.MiniMRCluster
+ createSubDir(
+ "mapred.local.dir",
+ testPath, "mapred-local-dir");
+
+ createSubDirAndSystemProperty(
+ "mapred.working.dir",
+ testPath, "mapred-working-dir");
}
private void createSubDir(String propertyName, Path parent, String subDirName){
@@ -428,24 +441,6 @@ public class HBaseTestingUtility {
return this.dfsCluster;
}
- /** This is used before starting HDFS and map-reduce mini-clusters */
- private void createDirsAndSetProperties() {
- setupClusterTestDir();
- System.setProperty(TEST_DIRECTORY_KEY, clusterTestDir.getPath());
- createDirAndSetProperty("hadoop_tmp", "hadoop.tmp.dir");
- createDirAndSetProperty("mapred_output", MapreduceTestingShim.getMROutputDirProp());
- createDirAndSetProperty("mapred_local", "mapred.local.dir");
- }
-
- private String createDirAndSetProperty(final String relPath, String property) {
- String path = clusterTestDir.getPath() + "/" + relPath;
- System.setProperty(property, path);
- conf.set(property, path);
- new File(path).mkdirs();
- LOG.info("Setting " + property + " to " + path + " in system properties and HBase conf");
- return path;
- }
-
/**
* Shuts down instance created by call to {@link #startMiniDFSCluster(int)}
* or does nothing.
@@ -1273,13 +1268,10 @@ public class HBaseTestingUtility {
conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
mrCluster = new MiniMRCluster(servers,
FileSystem.get(conf).getUri().toString(), 1);
- JobConf jobConf = mrCluster.createJobConf();
LOG.info("Mini mapreduce cluster started");
-
- // This fixes TestImportTsv but breaks TestImportExport tests
conf.set("mapred.job.tracker",
mrCluster.createJobConf().get("mapred.job.tracker"));
- // this for mrv2 support; mr1 ignores this
+ /* this for mrv2 support */
conf.set("mapreduce.framework.name", "yarn");
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java?rev=1357853&r1=1357852&r2=1357853&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/MapreduceTestingShim.java Thu Jul 5 19:58:54 2012
@@ -49,17 +49,11 @@ abstract public class MapreduceTestingSh
abstract public JobContext newJobContext(Configuration jobConf)
throws IOException;
- abstract public String obtainMROutputDirProp();
-
public static JobContext createJobContext(Configuration jobConf)
throws IOException {
return instance.newJobContext(jobConf);
}
- public static String getMROutputDirProp() {
- return instance.obtainMROutputDirProp();
- }
-
private static class MapreduceV1Shim extends MapreduceTestingShim {
public JobContext newJobContext(Configuration jobConf) throws IOException {
// Implementing:
@@ -74,11 +68,6 @@ abstract public class MapreduceTestingSh
"Failed to instantiate new JobContext(jobConf, new JobID())", e);
}
}
-
- @Override
- public String obtainMROutputDirProp() {
- return "mapred.output.dir";
- }
};
private static class MapreduceV2Shim extends MapreduceTestingShim {
@@ -94,12 +83,6 @@ abstract public class MapreduceTestingSh
"Failed to return from Job.getInstance(jobConf)");
}
}
-
- @Override
- public String obtainMROutputDirProp() {
- // o.a.h.mapreduce.lib.output FileOutputFormat.OUTDIR
- return "mapreduce.output.fileoutputformat.outputdir";
- }
};
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java?rev=1357853&r1=1357852&r2=1357853&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java Thu Jul 5 19:58:54 2012
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hbase.mapreduce;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
@@ -28,6 +27,7 @@ import org.apache.hadoop.hbase.HColumnDe
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
import org.junit.After;
@@ -45,6 +44,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import static org.junit.Assert.assertEquals;
@Category(MediumTests.class)
public class TestImportExport {
@@ -58,11 +58,12 @@ public class TestImportExport {
private static final byte[] QUAL = Bytes.toBytes("q");
private static final String OUTPUT_DIR = "outputdir";
+ private static MiniHBaseCluster cluster;
private static long now = System.currentTimeMillis();
@BeforeClass
public static void beforeClass() throws Exception {
- UTIL.startMiniCluster();
+ cluster = UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
}
@@ -104,16 +105,16 @@ public class TestImportExport {
"1000"
};
- GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+ GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
Configuration conf = opts.getConfiguration();
args = opts.getRemainingArgs();
Job job = Export.createSubmittableJob(conf, args);
job.getConfiguration().set("mapreduce.framework.name", "yarn");
job.waitForCompletion(false);
- HBaseFsck.debugLsr(conf, new Path("."));
assertTrue(job.isSuccessful());
+
String IMPORT_TABLE = "importTableSimpleCase";
t = UTIL.createTable(Bytes.toBytes(IMPORT_TABLE), FAMILYB);
args = new String[] {
@@ -122,14 +123,13 @@ public class TestImportExport {
OUTPUT_DIR
};
- opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+ opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
conf = opts.getConfiguration();
args = opts.getRemainingArgs();
job = Import.createSubmittableJob(conf, args);
job.getConfiguration().set("mapreduce.framework.name", "yarn");
job.waitForCompletion(false);
- HBaseFsck.debugLsr(conf, new Path("."));
assertTrue(job.isSuccessful());
Get g = new Get(ROW1);
@@ -174,7 +174,7 @@ public class TestImportExport {
"1000"
};
- GenericOptionsParser opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+ GenericOptionsParser opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
Configuration conf = opts.getConfiguration();
args = opts.getRemainingArgs();
@@ -198,7 +198,7 @@ public class TestImportExport {
OUTPUT_DIR
};
- opts = new GenericOptionsParser(new Configuration(UTIL.getConfiguration()), args);
+ opts = new GenericOptionsParser(new Configuration(cluster.getConfiguration()), args);
conf = opts.getConfiguration();
args = opts.getRemainingArgs();