You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/01 20:53:44 UTC
[02/11] git commit: HADOOP-11021. Configurable replication factor in
the hadoop archive command. Contributed by Zhe Zhang.
HADOOP-11021. Configurable replication factor in the hadoop archive command. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea1c6f31
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea1c6f31
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea1c6f31
Branch: refs/heads/HDFS-6581
Commit: ea1c6f31c2d2ea5b38ed57e2aa241d122103a721
Parents: c60da4d
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Aug 29 14:44:37 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Aug 29 14:44:37 2014 -0700
----------------------------------------------------------------------
.../src/site/markdown/HadoopArchives.md.vm | 12 ++++--
.../org/apache/hadoop/tools/HadoopArchives.java | 12 +++++-
.../apache/hadoop/tools/TestHadoopArchives.java | 41 ++++++++++++++++++++
3 files changed, 59 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1c6f31/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm
index 0cc0f1c..db0a25f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/HadoopArchives.md.vm
@@ -38,7 +38,7 @@ Overview
How to Create an Archive
------------------------
- `Usage: hadoop archive -archiveName name -p <parent> <src>* <dest>`
+ `Usage: hadoop archive -archiveName name -p <parent> [-r <replication factor>] <src>* <dest>`
-archiveName is the name of the archive you would like to create. An example
would be foo.har. The name should have a \*.har extension. The parent argument
@@ -52,9 +52,12 @@ How to Create an Archive
would need a map reduce cluster to run this. For a detailed example the later
sections.
+ -r indicates the desired replication factor; if this optional argument is
+ not specified, a replication factor of 10 will be used.
+
If you just want to archive a single directory /foo/bar then you can just use
- `hadoop archive -archiveName zoo.har -p /foo/bar /outputdir`
+ `hadoop archive -archiveName zoo.har -p /foo/bar -r 3 /outputdir`
How to Look Up Files in Archives
--------------------------------
@@ -90,14 +93,15 @@ Archives Examples
$H3 Creating an Archive
- `hadoop archive -archiveName foo.har -p /user/hadoop dir1 dir2 /user/zoo`
+ `hadoop archive -archiveName foo.har -p /user/hadoop -r 3 dir1 dir2 /user/zoo`
The above example is creating an archive using /user/hadoop as the relative
archive directory. The directories /user/hadoop/dir1 and /user/hadoop/dir2
will be archived in the following file system directory -- /user/zoo/foo.har.
Archiving does not delete the input files. If you want to delete the input
files after creating the archives (to reduce namespace), you will have to do
- it on your own.
+ it on your own. In this example, because `-r 3` is specified, a replication
+ factor of 3 will be used.
$H3 Looking Up Files
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1c6f31/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
index 93994b8..e53576d 100644
--- a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
@@ -97,9 +97,12 @@ public class HadoopArchives implements Tool {
long partSize = 2 * 1024 * 1024 * 1024l;
/** size of blocks in hadoop archives **/
long blockSize = 512 * 1024 * 1024l;
+ /** the desired replication degree; default is 10 **/
+ short repl = 10;
private static final String usage = "archive"
- + " -archiveName NAME -p <parent path> <src>* <dest>" +
+ + " -archiveName NAME -p <parent path> [-r <replication factor>]" +
+ "<src>* <dest>" +
"\n";
@@ -542,7 +545,7 @@ public class HadoopArchives implements Tool {
srcWriter.close();
}
//increase the replication of src files
- jobfs.setReplication(srcFiles, (short) 10);
+ jobfs.setReplication(srcFiles, repl);
conf.setInt(SRC_COUNT_LABEL, numFiles);
conf.setLong(TOTAL_SIZE_LABEL, totalSize);
int numMaps = (int)(totalSize/partSize);
@@ -835,6 +838,11 @@ public class HadoopArchives implements Tool {
}
i+=2;
+
+ if ("-r".equals(args[i])) {
+ repl = Short.parseShort(args[i+1]);
+ i+=2;
+ }
//read the rest of the paths
for (; i < args.length; i++) {
if (i == (args.length - 1)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea1c6f31/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
index 65bbbe4..e7eef3f 100644
--- a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
@@ -157,6 +157,24 @@ public class TestHadoopArchives {
final List<String> harPaths = lsr(shell, fullHarPathStr);
Assert.assertEquals(originalPaths, harPaths);
}
+
+ @Test
+ public void testRelativePathWitRepl() throws Exception {
+ final Path sub1 = new Path(inputPath, "dir1");
+ fs.mkdirs(sub1);
+ createFile(inputPath, fs, sub1.getName(), "a");
+ final FsShell shell = new FsShell(conf);
+
+ final List<String> originalPaths = lsr(shell, "input");
+ System.out.println("originalPaths: " + originalPaths);
+
+ // make the archive:
+ final String fullHarPathStr = makeArchiveWithRepl();
+
+ // compare results:
+ final List<String> harPaths = lsr(shell, fullHarPathStr);
+ Assert.assertEquals(originalPaths, harPaths);
+ }
@Test
public void testPathWithSpaces() throws Exception {
@@ -625,6 +643,29 @@ public class TestHadoopArchives {
assertEquals(0, ToolRunner.run(har, args));
return fullHarPathStr;
}
+
+ /*
+ * Run the HadoopArchives tool to create an archive on the
+ * given file system with a specified replication degree.
+ */
+ private String makeArchiveWithRepl() throws Exception {
+ final String inputPathStr = inputPath.toUri().getPath();
+ System.out.println("inputPathStr = " + inputPathStr);
+
+ final URI uri = fs.getUri();
+ final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort()
+ + archivePath.toUri().getPath() + Path.SEPARATOR;
+
+ final String harName = "foo.har";
+ final String fullHarPathStr = prefix + harName;
+ final String[] args = { "-archiveName", harName, "-p", inputPathStr,
+ "-r 3", "*", archivePath.toString() };
+ System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
+ HADOOP_ARCHIVES_JAR);
+ final HadoopArchives har = new HadoopArchives(conf);
+ assertEquals(0, ToolRunner.run(har, args));
+ return fullHarPathStr;
+ }
@Test
/*