You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by op...@apache.org on 2018/11/10 02:18:56 UTC
hbase git commit: HBASE-21445 CopyTable by bulkload will write hfile
into yarn's HDFS
Repository: hbase
Updated Branches:
refs/heads/master ccabf7310 -> 813528550
HBASE-21445 CopyTable by bulkload will write hfile into yarn's HDFS
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/81352855
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/81352855
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/81352855
Branch: refs/heads/master
Commit: 81352855068d2f81fe1f08967ed0de6d2873136d
Parents: ccabf73
Author: huzheng <op...@gmail.com>
Authored: Wed Nov 7 11:25:43 2018 +0800
Committer: huzheng <op...@gmail.com>
Committed: Sat Nov 10 10:18:01 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapreduce/CopyTable.java | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/81352855/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index b20f07f..4e57f54 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -139,13 +140,13 @@ public class CopyTable extends Configured implements Tool {
job.setNumReduceTasks(0);
if (bulkload) {
- TableMapReduceUtil.initTableMapperJob(tableName, scan, Import.CellImporter.class, null,
- null, job);
+ TableMapReduceUtil.initTableMapperJob(tableName, scan, Import.CellImporter.class, null, null,
+ job);
// We need to split the inputs by destination tables so that output of Map can be bulk-loaded.
TableInputFormat.configureSplitTable(job, TableName.valueOf(dstTableName));
- FileSystem fs = FileSystem.get(getConf());
+ FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
Random rand = new Random();
Path root = new Path(fs.getWorkingDirectory(), "copytable");
fs.mkdirs(root);
@@ -161,7 +162,7 @@ public class CopyTable extends Configured implements Tool {
try (Connection conn = ConnectionFactory.createConnection(getConf());
Admin admin = conn.getAdmin()) {
HFileOutputFormat2.configureIncrementalLoadMap(job,
- admin.getDescriptor((TableName.valueOf(dstTableName))));
+ admin.getDescriptor((TableName.valueOf(dstTableName))));
}
} else {
TableMapReduceUtil.initTableMapperJob(tableName, scan,
@@ -370,12 +371,12 @@ public class CopyTable extends Configured implements Tool {
}
int code = 0;
if (bulkload) {
- code = new LoadIncrementalHFiles(this.getConf()).run(new String[]{this.bulkloadDir.toString(),
- this.dstTableName});
+ code = new LoadIncrementalHFiles(this.getConf())
+ .run(new String[] { this.bulkloadDir.toString(), this.dstTableName });
if (code == 0) {
// bulkloadDir is deleted only LoadIncrementalHFiles was successful so that one can rerun
// LoadIncrementalHFiles.
- FileSystem fs = FileSystem.get(this.getConf());
+ FileSystem fs = FSUtils.getCurrentFileSystem(getConf());
if (!fs.delete(this.bulkloadDir, true)) {
LOG.error("Deleting folder " + bulkloadDir + " failed!");
code = 1;