You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2013/11/15 22:54:01 UTC
svn commit: r1542414 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Author: ndimiduk
Date: Fri Nov 15 21:54:00 2013
New Revision: 1542414
URL: http://svn.apache.org/r1542414
Log:
HBASE-9165 [mapreduce] Modularize building dependency jars
Separate adding HBase and dependencies from adding other job dependencies, and
expose it as a separate method that other projects can use (for PIG-3285,
HIVE-2055).
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java?rev=1542414&r1=1542413&r2=1542414&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java Fri Nov 15 21:54:00 2013
@@ -294,14 +294,13 @@ public class TableMapReduceUtil {
}
/**
- * @see org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil#addDependencyJars(Job)
+ * @see org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)
*/
public static void addDependencyJars(JobConf job) throws IOException {
+ org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job);
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(
job,
- org.apache.zookeeper.ZooKeeper.class,
- com.google.common.base.Function.class,
- com.google.protobuf.Message.class,
+ // when making changes here, consider also mapreduce.TableMapReduceUtil
job.getMapOutputKeyClass(),
job.getMapOutputValueClass(),
job.getOutputKeyClass(),
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java?rev=1542414&r1=1542413&r2=1542414&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java Fri Nov 15 21:54:00 2013
@@ -552,25 +552,44 @@ public static void initCredentials(Job j
}
/**
+ * Add HBase and its dependencies (only) to the job configuration.
+ * <p>
+ * This is intended as a low-level API, facilitating code reuse between this
+ * class and its mapred counterpart. It also of use to extenral tools that
+ * need to build a MapReduce job that interacts with HBase but want
+ * fine-grained control over the jars shipped to the cluster.
+ * </p>
+ * @param conf The Configuration object to extend with dependencies.
+ * @see org.apache.hadoop.hbase.mapred.TableMapReduceUtil
+ * @see <a href="https://issues.apache.org/jira/browse/PIG-3285">PIG-3285</a>
+ */
+ public static void addHBaseDependencyJars(Configuration conf) throws IOException {
+ addDependencyJars(conf,
+ org.apache.zookeeper.ZooKeeper.class,
+ com.google.protobuf.Message.class,
+ com.google.common.base.Function.class,
+ com.google.common.collect.ImmutableSet.class,
+ org.apache.hadoop.hbase.util.Bytes.class); //one class from hbase.jar
+ }
+
+ /**
* Add the HBase dependency jars as well as jars for any of the configured
* job classes to the job configuration, so that JobClient will ship them
* to the cluster and add them to the DistributedCache.
*/
public static void addDependencyJars(Job job) throws IOException {
+ addHBaseDependencyJars(job.getConfiguration());
try {
addDependencyJars(job.getConfiguration(),
- org.apache.zookeeper.ZooKeeper.class,
- com.google.protobuf.Message.class,
- com.google.common.collect.ImmutableSet.class,
- org.apache.hadoop.hbase.util.Bytes.class, //one class from hbase.jar
- job.getMapOutputKeyClass(),
- job.getMapOutputValueClass(),
- job.getInputFormatClass(),
- job.getOutputKeyClass(),
- job.getOutputValueClass(),
- job.getOutputFormatClass(),
- job.getPartitionerClass(),
- job.getCombinerClass());
+ // when making changes here, consider also mapred.TableMapReduceUtil
+ job.getMapOutputKeyClass(),
+ job.getMapOutputValueClass(),
+ job.getInputFormatClass(),
+ job.getOutputKeyClass(),
+ job.getOutputValueClass(),
+ job.getOutputFormatClass(),
+ job.getPartitionerClass(),
+ job.getCombinerClass());
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
@@ -612,8 +631,7 @@ public static void initCredentials(Job j
}
if (jars.isEmpty()) return;
- conf.set("tmpjars",
- StringUtils.arrayToString(jars.toArray(new String[0])));
+ conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0])));
}
/**
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java?rev=1542414&r1=1542413&r2=1542414&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java Fri Nov 15 21:54:00 2013
@@ -263,19 +263,15 @@ public class TestTableMapReduce {
/**
* Test that we add tmpjars correctly including the ZK jar.
*/
+ @Test
public void testAddDependencyJars() throws Exception {
Job job = new Job();
TableMapReduceUtil.addDependencyJars(job);
String tmpjars = job.getConfiguration().get("tmpjars");
- System.err.println("tmpjars: " + tmpjars);
+ assertTrue(tmpjars.contains("hbase"));
assertTrue(tmpjars.contains("zookeeper"));
- assertFalse(tmpjars.contains("guava"));
-
- System.err.println("appending guava jar");
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
- com.google.common.base.Function.class);
- tmpjars = job.getConfiguration().get("tmpjars");
+ assertTrue(tmpjars.contains("protobuf"));
assertTrue(tmpjars.contains("guava"));
}