You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/08/17 18:34:18 UTC
[12/50] [abbrv] hbase git commit: HBASE-8386 deprecate
TableMapReduce.addDependencyJars(Configuration, class ...)
HBASE-8386 deprecate TableMapReduce.addDependencyJars(Configuration, class<?> ...)
Signed-off-by: Matteo Bertozzi <ma...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/431c8c9a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/431c8c9a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/431c8c9a
Branch: refs/heads/hbase-12439
Commit: 431c8c9ad03336b5ab844d97adc47b8a73d7aa00
Parents: c59f764
Author: Sean Busbey <bu...@apache.org>
Authored: Tue Jul 19 07:21:26 2016 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Aug 5 10:35:02 2016 -0500
----------------------------------------------------------------------
.../test/IntegrationTestBigLinkedList.java | 6 ++--
.../test/IntegrationTestLoadAndVerify.java | 4 +--
...tionTestWithCellVisibilityLoadAndVerify.java | 2 +-
.../hadoop/hbase/mapred/TableMapReduceUtil.java | 2 +-
.../apache/hadoop/hbase/mapreduce/Import.java | 6 ++--
.../hadoop/hbase/mapreduce/ImportTsv.java | 2 +-
.../hbase/mapreduce/TableMapReduceUtil.java | 30 ++++++++++++++++++--
.../hadoop/hbase/mapreduce/WALPlayer.java | 4 +--
.../hadoop/hbase/PerformanceEvaluation.java | 2 +-
.../mapred/TestTableSnapshotInputFormat.java | 2 +-
.../mapreduce/TestTableSnapshotInputFormat.java | 2 +-
src/main/asciidoc/_chapters/mapreduce.adoc | 2 +-
12 files changed, 45 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
index 6c54dca..135bea7 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
@@ -785,7 +785,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
TableMapReduceUtil.addDependencyJars(job);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
+ AbstractHBaseTool.class);
TableMapReduceUtil.initCredentials(job);
boolean success = jobCompletion(job);
@@ -1296,7 +1297,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
+ AbstractHBaseTool.class);
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
index e279dfb..d12383d 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
@@ -348,7 +348,7 @@ public void cleanUpCluster() throws Exception {
TableMapReduceUtil.addDependencyJars(job);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
TableMapReduceUtil.initCredentials(job);
assertTrue(job.waitForCompletion(true));
return job;
@@ -372,7 +372,7 @@ public void cleanUpCluster() throws Exception {
TableMapReduceUtil.initTableMapperJob(
htd.getTableName().getNameAsString(), scan, VerifyMapper.class,
BytesWritable.class, BytesWritable.class, job);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
TableMapReduceUtil.setScannerCaching(job, scannerCaching);
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
index 52a705b..abaa481 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
@@ -331,7 +331,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT
scan.setAuthorizations(new Authorizations(auths));
TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan,
VerifyMapper.class, NullWritable.class, NullWritable.class, job);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
TableMapReduceUtil.setScannerCaching(job, scannerCaching);
job.setNumReduceTasks(0);
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
index 84a279d..73340c4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
@@ -362,7 +362,7 @@ public class TableMapReduceUtil {
*/
public static void addDependencyJars(JobConf job) throws IOException {
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job);
- org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(
+ org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(
job,
// when making changes here, consider also mapreduce.TableMapReduceUtil
// pull job classes
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index c7d5912..e2693b9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -616,7 +616,7 @@ public class Import extends Configured implements Tool {
try {
Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
if (filter != null) {
- TableMapReduceUtil.addDependencyJars(conf, filter);
+ TableMapReduceUtil.addDependencyJarsForClasses(conf, filter);
}
} catch (Exception e) {
throw new IOException(e);
@@ -643,7 +643,7 @@ public class Import extends Configured implements Tool {
fs.deleteOnExit(partitionsPath);
job.setPartitionerClass(KeyValueWritableComparablePartitioner.class);
job.setNumReduceTasks(regionLocator.getStartKeys().length);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Preconditions.class);
}
} else if (hfileOutPath != null) {
@@ -658,7 +658,7 @@ public class Import extends Configured implements Tool {
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(KeyValue.class);
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Preconditions.class);
}
} else {
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index e778d99..39085df 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -587,7 +587,7 @@ public class ImportTsv extends Configured implements Tool {
KeyValueSerialization.class.getName());
}
TableMapReduceUtil.addDependencyJars(job);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Function.class /* Guava used by TsvParser */);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 37e4e44..8d8a784 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -339,7 +339,7 @@ public class TableMapReduceUtil {
if (addDependencyJars) {
addDependencyJars(job);
- addDependencyJars(job.getConfiguration(), MetricRegistry.class);
+ addDependencyJarsForClasses(job.getConfiguration(), MetricRegistry.class);
}
resetCacheConfig(job.getConfiguration());
@@ -787,7 +787,7 @@ public class TableMapReduceUtil {
" Continuing without it.");
}
- addDependencyJars(conf,
+ addDependencyJarsForClasses(conf,
// explicitly pull a class from each module
org.apache.hadoop.hbase.HConstants.class, // hbase-common
org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
@@ -835,7 +835,7 @@ public class TableMapReduceUtil {
public static void addDependencyJars(Job job) throws IOException {
addHBaseDependencyJars(job.getConfiguration());
try {
- addDependencyJars(job.getConfiguration(),
+ addDependencyJarsForClasses(job.getConfiguration(),
// when making changes here, consider also mapred.TableMapReduceUtil
// pull job classes
job.getMapOutputKeyClass(),
@@ -855,9 +855,33 @@ public class TableMapReduceUtil {
* Add the jars containing the given classes to the job's configuration
* such that JobClient will ship them to the cluster and add them to
* the DistributedCache.
+ * @deprecated rely on {@link #addDependencyJars(Job)} instead.
*/
+ @Deprecated
public static void addDependencyJars(Configuration conf,
Class<?>... classes) throws IOException {
+ LOG.warn("The addDependencyJars(Configuration, Class<?>...) method has been deprecated since it"
+ + " is easy to use incorrectly. Most users should rely on addDependencyJars(Job) " +
+ "instead. See HBASE-8386 for more details.");
+ addDependencyJarsForClasses(conf, classes);
+ }
+
+ /**
+ * Add the jars containing the given classes to the job's configuration
+ * such that JobClient will ship them to the cluster and add them to
+ * the DistributedCache.
+ *
+ * N.B. that this method at most adds one jar per class given. If there is more than one
+ * jar available containing a class with the same name as a given class, we don't define
+ * which of those jars might be chosen.
+ *
+ * @param conf The Hadoop Configuration to modify
+ * @param classes will add just those dependencies needed to find the given classes
+ * @throws IOException if an underlying library call fails.
+ */
+ @InterfaceAudience.Private
+ public static void addDependencyJarsForClasses(Configuration conf,
+ Class<?>... classes) throws IOException {
FileSystem localFs = FileSystem.getLocal(conf);
Set<String> jars = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index c6fefb2..452714b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -282,7 +282,7 @@ public class WALPlayer extends Configured implements Tool {
RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
}
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Preconditions.class);
} else {
// output to live cluster
@@ -295,7 +295,7 @@ public class WALPlayer extends Configured implements Tool {
}
String codecCls = WALCellCodec.getWALCellCodecClass(conf);
try {
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Class.forName(codecCls));
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Class.forName(codecCls));
} catch (Exception e) {
throw new IOException("Cannot determine wal codec class " + codecCls, e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 0528975..dc8a783 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -499,7 +499,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
TableMapReduceUtil.addDependencyJars(job);
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
Histogram.class, // yammer metrics
ObjectMapper.class); // jackson-mapper-asl
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
index c85cead..c4fdea4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
@@ -243,7 +243,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
JobConf jobConf = new JobConf(util.getConfiguration());
jobConf.setJarByClass(util.getClass());
- org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
+ org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(jobConf,
TestTableSnapshotInputFormat.class);
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS,
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
index 7c10def..5e8c8cd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
@@ -281,7 +281,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
Scan scan = new Scan(startRow, endRow); // limit the scan
job.setJarByClass(util.getClass());
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
TestTableSnapshotInputFormat.class);
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
http://git-wip-us.apache.org/repos/asf/hbase/blob/431c8c9a/src/main/asciidoc/_chapters/mapreduce.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/mapreduce.adoc b/src/main/asciidoc/_chapters/mapreduce.adoc
index 75718fd..dfa843a 100644
--- a/src/main/asciidoc/_chapters/mapreduce.adoc
+++ b/src/main/asciidoc/_chapters/mapreduce.adoc
@@ -73,7 +73,7 @@ This example assumes you use a BASH-compatible shell.
$ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar rowcounter usertable
----
-When the command runs, internally, the HBase JAR finds the dependencies it needs for ZooKeeper, Guava, and its other dependencies on the passed `HADOOP_CLASSPATH` and adds the JARs to the MapReduce job configuration.
+When the command runs, internally, the HBase JAR finds the dependencies it needs and adds them to the MapReduce job configuration.
See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)` for how this is done.
The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce, which are the same jars `TableMapReduceUtil#addDependencyJars` would add.