You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2015/05/06 16:29:19 UTC
hbase git commit: HBASE-13625 Use HDFS for HFileOutputFormat2
partitioner's path (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/branch-1 ad8f1d076 -> ec3d7189a
HBASE-13625 Use HDFS for HFileOutputFormat2 partitioner's path (Stephen Yuan Jiang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ec3d7189
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ec3d7189
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ec3d7189
Branch: refs/heads/branch-1
Commit: ec3d7189ad98e96a3e819687d192c025a09078af
Parents: ad8f1d0
Author: tedyu <yu...@gmail.com>
Authored: Wed May 6 07:29:57 2015 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed May 6 07:29:57 2015 -0700
----------------------------------------------------------------------
.../hadoop/hbase/security/SecureBulkLoadUtil.java | 2 +-
hbase-common/src/main/resources/hbase-default.xml | 14 ++++++++++++++
.../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 2 +-
.../hadoop/hbase/mapreduce/TestHFileOutputFormat.java | 5 ++++-
.../hbase/mapreduce/TestHFileOutputFormat2.java | 5 ++++-
5 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec3d7189/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java
index 2fde925..04bfbb5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java
@@ -37,6 +37,6 @@ public class SecureBulkLoadUtil {
}
public static Path getBaseStagingDir(Configuration conf) {
- return new Path(conf.get(BULKLOAD_STAGING_DIR, "/tmp/hbase-staging"));
+ return new Path(conf.get(BULKLOAD_STAGING_DIR));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec3d7189/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 86a5104..2d05e96 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -63,6 +63,20 @@ possible configurations would overwhelm and obscure the important.
machine restart.</description>
</property>
<property >
+ <name>hbase.fs.tmp.dir</name>
+ <value>/user/${user.name}/hbase-staging</value>
+ <description>A staging directory in default file system (HDFS)
+ for keeping temporary data.
+ </description>
+ </property>
+ <property >
+ <name>hbase.bulkload.staging.dir</name>
+ <value>${hbase.fs.tmp.dir}</value>
+ <description>A staging directory in default file system (HDFS)
+ for bulk loading.
+ </description>
+ </property>
+ <property >
<name>hbase.cluster.distributed</name>
<value>false</value>
<description>The mode the cluster will be in. Possible values are
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec3d7189/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index e533bf2..678b7bb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -588,7 +588,7 @@ public class HFileOutputFormat2
Configuration conf = job.getConfiguration();
// create the partitions file
FileSystem fs = FileSystem.get(conf);
- Path partitionsPath = new Path(conf.get("hadoop.tmp.dir"), "partitions_" + UUID.randomUUID());
+ Path partitionsPath = new Path(conf.get("hbase.fs.tmp.dir"), "partitions_" + UUID.randomUUID());
fs.makeQualified(partitionsPath);
writePartitions(conf, partitionsPath, splitPoints);
fs.deleteOnExit(partitionsPath);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec3d7189/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
index 5e278f8..f3a5c4c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
@@ -333,7 +333,9 @@ public class TestHFileOutputFormat {
@Test
public void testJobConfiguration() throws Exception {
- Job job = new Job(util.getConfiguration());
+ Configuration conf = new Configuration(this.util.getConfiguration());
+ conf.set("hbase.fs.tmp.dir", util.getDataTestDir("testJobConfiguration").toString());
+ Job job = new Job(conf);
job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration"));
HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
RegionLocator regionLocator = Mockito.mock(RegionLocator.class);
@@ -820,6 +822,7 @@ public class TestHFileOutputFormat {
// We turn off the sequence file compression, because DefaultCodec
// pollutes the GZip codec pool with an incompatible compressor.
conf.set("io.seqfile.compression.type", "NONE");
+ conf.set("hbase.fs.tmp.dir", dir.toString());
Job job = new Job(conf, "testLocalMRIncrementalLoad");
job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings"));
setupRandomGeneratorMapper(job);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec3d7189/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index d593b25..d34cfc1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -337,7 +337,9 @@ public class TestHFileOutputFormat2 {
@Test
public void testJobConfiguration() throws Exception {
- Job job = new Job(util.getConfiguration());
+ Configuration conf = new Configuration(this.util.getConfiguration());
+ conf.set("hbase.fs.tmp.dir", util.getDataTestDir("testJobConfiguration").toString());
+ Job job = new Job(conf);
job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration"));
RegionLocator regionLocator = Mockito.mock(RegionLocator.class);
setupMockStartKeys(regionLocator);
@@ -822,6 +824,7 @@ public class TestHFileOutputFormat2 {
// We turn off the sequence file compression, because DefaultCodec
// pollutes the GZip codec pool with an incompatible compressor.
conf.set("io.seqfile.compression.type", "NONE");
+ conf.set("hbase.fs.tmp.dir", dir.toString());
Job job = new Job(conf, "testLocalMRIncrementalLoad");
job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings"));
setupRandomGeneratorMapper(job);