You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:22:28 UTC
svn commit: r1077512 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
core/org/apache/hadoop/fs/ core/org/apache/hadoop/util/
docs/src/documentation/content/xdocs/ mapred/org/apache/hadoop/filecache/
mapred/org/apache/hadoop/mapred/ test/...
Author: omalley
Date: Fri Mar 4 04:22:27 2011
New Revision: 1077512
URL: http://svn.apache.org/viewvc?rev=1077512&view=rev
Log:
commit 0460135a6f0d4c2d0b967c88da7ea40a6329e284
Author: Amareshwari Sri Ramadasu <am...@yahoo-inc.com>
Date: Thu Jun 24 09:21:28 2010 +0530
MAPREDUCE-787. Fix JobSubmitter to honor user given symlink in the path.
+++ b/YAHOO-CHANGES.txt
+ MAPREDUCE-787. Fix JobSubmitter to honor user given symlink in the path.
+ (amareshwari)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java
hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/streaming.xml
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/filecache/TaskDistributedCacheManager.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestPath.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestGenericOptionsParser.java
hadoop/common/branches/branch-0.20-security-patches/src/test/testshell/ExternalMapReduce.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/Path.java Fri Mar 4 04:22:27 2011
@@ -63,13 +63,13 @@ public class Path implements Comparable
if (!(parentPath.equals("/") || parentPath.equals("")))
try {
parentUri = new URI(parentUri.getScheme(), parentUri.getAuthority(),
- parentUri.getPath()+"/", null, null);
+ parentUri.getPath()+"/", null, parentUri.getFragment());
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
URI resolved = parentUri.resolve(child.uri);
initialize(resolved.getScheme(), resolved.getAuthority(),
- normalizePath(resolved.getPath()));
+ normalizePath(resolved.getPath()), resolved.getFragment());
}
private void checkPathArg( String path ) {
@@ -123,18 +123,26 @@ public class Path implements Comparable
// uri path is the rest of the string -- query & fragment not supported
String path = pathString.substring(start, pathString.length());
- initialize(scheme, authority, path);
+ initialize(scheme, authority, path, null);
}
/** Construct a Path from components. */
public Path(String scheme, String authority, String path) {
checkPathArg( path );
- initialize(scheme, authority, path);
+ initialize(scheme, authority, path, null);
}
- private void initialize(String scheme, String authority, String path) {
+ /**
+ * Construct a path from a URI
+ */
+ public Path(URI aUri) {
+ uri = aUri;
+ }
+
+ private void initialize(String scheme, String authority, String path,
+ String fragment) {
try {
- this.uri = new URI(scheme, authority, normalizePath(path), null, null)
+ this.uri = new URI(scheme, authority, normalizePath(path), null, fragment)
.normalize();
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
@@ -233,6 +241,10 @@ public class Path implements Comparable
path = path.substring(1); // remove slash before drive
buffer.append(path);
}
+ if (uri.getFragment() != null) {
+ buffer.append("#");
+ buffer.append(uri.getFragment());
+ }
return buffer.toString();
}
@@ -277,7 +289,7 @@ public class Path implements Comparable
String scheme = pathUri.getScheme();
String authority = pathUri.getAuthority();
-
+ String fragment = pathUri.getFragment();
if (scheme != null &&
(authority != null || fsUri.getAuthority() == null))
return path;
@@ -292,7 +304,14 @@ public class Path implements Comparable
authority = "";
}
}
-
- return new Path(scheme+":"+"//"+authority + pathUri.getPath());
+
+ URI newUri = null;
+ try {
+ newUri = new URI(scheme, authority ,
+ normalizePath(pathUri.getPath()), null, fragment);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ return new Path(newUri);
}
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/GenericOptionsParser.java Fri Mar 4 04:22:27 2011
@@ -21,8 +21,11 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -111,16 +114,20 @@ public class GenericOptionsParser {
* Create an options parser with the given options to parse the args.
* @param opts the options
* @param args the command line arguments
+ * @throws IOException
*/
- public GenericOptionsParser(Options opts, String[] args) {
+ public GenericOptionsParser(Options opts, String[] args)
+ throws IOException {
this(new Configuration(), new Options(), args);
}
/**
* Create an options parser to parse the args.
* @param args the command line arguments
+ * @throws IOException
*/
- public GenericOptionsParser(String[] args) {
+ public GenericOptionsParser(String[] args)
+ throws IOException {
this(new Configuration(), new Options(), args);
}
@@ -133,8 +140,10 @@ public class GenericOptionsParser {
*
* @param conf the <code>Configuration</code> to modify.
* @param args command-line arguments.
+ * @throws IOException
*/
- public GenericOptionsParser(Configuration conf, String[] args) {
+ public GenericOptionsParser(Configuration conf, String[] args)
+ throws IOException {
this(conf, new Options(), args);
}
@@ -148,8 +157,10 @@ public class GenericOptionsParser {
* @param conf the configuration to modify
* @param options options built by the caller
* @param args User-specified arguments
+ * @throws IOException
*/
- public GenericOptionsParser(Configuration conf, Options options, String[] args) {
+ public GenericOptionsParser(Configuration conf,
+ Options options, String[] args) throws IOException {
parseGeneralOptions(options, conf, args);
this.conf = conf;
}
@@ -246,7 +257,7 @@ public class GenericOptionsParser {
* @param line User-specified generic options
*/
private void processGeneralOptions(Configuration conf,
- CommandLine line) {
+ CommandLine line) throws IOException {
if (line.hasOption("fs")) {
FileSystem.setDefaultUri(conf, line.getOptionValue("fs"));
}
@@ -260,29 +271,25 @@ public class GenericOptionsParser {
conf.addResource(new Path(value));
}
}
- try {
- if (line.hasOption("libjars")) {
- conf.set("tmpjars",
- validateFiles(line.getOptionValue("libjars"), conf));
- //setting libjars in client classpath
- URL[] libjars = getLibJars(conf);
- if(libjars!=null && libjars.length>0) {
- conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
- Thread.currentThread().setContextClassLoader(
- new URLClassLoader(libjars,
- Thread.currentThread().getContextClassLoader()));
- }
- }
- if (line.hasOption("files")) {
- conf.set("tmpfiles",
- validateFiles(line.getOptionValue("files"), conf));
- }
- if (line.hasOption("archives")) {
- conf.set("tmparchives",
- validateFiles(line.getOptionValue("archives"), conf));
+ if (line.hasOption("libjars")) {
+ conf.set("tmpjars",
+ validateFiles(line.getOptionValue("libjars"), conf));
+ //setting libjars in client classpath
+ URL[] libjars = getLibJars(conf);
+ if(libjars!=null && libjars.length>0) {
+ conf.setClassLoader(new URLClassLoader(libjars, conf.getClassLoader()));
+ Thread.currentThread().setContextClassLoader(
+ new URLClassLoader(libjars,
+ Thread.currentThread().getContextClassLoader()));
}
- } catch (IOException ioe) {
- System.err.println(StringUtils.stringifyException(ioe));
+ }
+ if (line.hasOption("files")) {
+ conf.set("tmpfiles",
+ validateFiles(line.getOptionValue("files"), conf));
+ }
+ if (line.hasOption("archives")) {
+ conf.set("tmparchives",
+ validateFiles(line.getOptionValue("archives"), conf));
}
if (line.hasOption('D')) {
String[] property = line.getOptionValues('D');
@@ -328,12 +335,14 @@ public class GenericOptionsParser {
return null;
}
String[] files = jars.split(",");
- URL[] cp = new URL[files.length];
- for (int i=0;i<cp.length;i++) {
- Path tmp = new Path(files[i]);
- cp[i] = FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL();
+ List<URL> cp = new ArrayList<URL>();
+ for (String file : files) {
+ Path tmp = new Path(file);
+ if (tmp.getFileSystem(conf).equals(FileSystem.getLocal(conf))) {
+ cp.add(FileSystem.getLocal(conf).pathToFile(tmp).toURI().toURL());
+ }
}
- return cp;
+ return cp.toArray(new URL[0]);
}
/**
@@ -346,7 +355,8 @@ public class GenericOptionsParser {
* @param files
* @return
*/
- private String validateFiles(String files, Configuration conf) throws IOException {
+ private String validateFiles(String files, Configuration conf)
+ throws IOException {
if (files == null)
return null;
String[] fileArr = files.split(",");
@@ -354,8 +364,13 @@ public class GenericOptionsParser {
for (int i =0; i < fileArr.length; i++) {
String tmp = fileArr[i];
String finalPath;
- Path path = new Path(tmp);
- URI pathURI = path.toUri();
+ URI pathURI;
+ try {
+ pathURI = new URI(tmp);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ Path path = new Path(pathURI);
FileSystem localFs = FileSystem.getLocal(conf);
if (pathURI.getScheme() == null) {
//default to the local file system
@@ -375,9 +390,6 @@ public class GenericOptionsParser {
throw new FileNotFoundException("File " + tmp + " does not exist.");
}
finalPath = path.makeQualified(fs).toString();
- try {
- fs.close();
- } catch(IOException e){};
}
finalArr[i] = finalPath;
}
@@ -393,7 +405,7 @@ public class GenericOptionsParser {
* @return Command-specific arguments
*/
private String[] parseGeneralOptions(Options opts, Configuration conf,
- String[] args) {
+ String[] args) throws IOException {
opts = buildGeneralOptions(opts);
CommandLineParser parser = new GnuParser();
try {
Modified: hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/mapred_tutorial.xml Fri Mar 4 04:22:27 2011
@@ -604,17 +604,36 @@
would be present in the current working directory of the task
using the option <code>-files</code>. The <code>-libjars</code>
option allows applications to add jars to the classpaths of the maps
- and reduces. The <code>-archives</code> allows them to pass archives
- as arguments that are unzipped/unjarred and a link with name of the
- jar/zip are created in the current working directory of tasks. More
+ and reduces. The option <code>-archives</code> allows them to pass
+ comma separated list of archives as arguments. These archives are
+ unarchived and a link with name of the archive is created in
+ the current working directory of tasks. More
details about the command line options are available at
<a href="commands_manual.html">Commands Guide.</a></p>
<p>Running <code>wordcount</code> example with
- <code>-libjars</code> and <code>-files</code>:<br/>
+ <code>-libjars</code>, <code>-files</code> and <code>-archives</code>:
+ <br/>
<code> hadoop jar hadoop-examples.jar wordcount -files cachefile.txt
- -libjars mylib.jar input output </code>
- </p>
+ -libjars mylib.jar -archives myarchive.zip input output </code>
+ Here, myarchive.zip will be placed and unzipped into a directory
+ by the name "myarchive.zip".
+ </p>
+
+ <p> Users can specify a different symbolic name for
+ files and archives passed through -files and -archives option, using #.
+ </p>
+
+ <p> For example,
+ <code> hadoop jar hadoop-examples.jar wordcount
+ -files dir1/dict.txt#dict1,dir2/dict.txt#dict2
+ -archives mytar.tgz#tgzdir input output </code>
+ Here, the files dir1/dict.txt and dir2/dict.txt can be accessed by
+ tasks using the symbolic names dict1 and dict2 respectively.
+ The archive mytar.tgz will be placed and unarchived into a
+ directory by the name "tgzdir".
+ </p>
+
</section>
<section>
Modified: hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/streaming.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/streaming.xml?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/streaming.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/docs/src/documentation/content/xdocs/streaming.xml Fri Mar 4 04:22:27 2011
@@ -317,6 +317,10 @@ This symlink points to the local copy of
<source>
-files hdfs://host:fs_port/user/testfile.txt
</source>
+ <p> User can specify a different symlink name for -files using #. </p>
+<source>
+-files hdfs://host:fs_port/user/testfile.txt#testfile
+</source>
<p>
Multiple entries can be specified like this:
</p>
@@ -337,6 +341,10 @@ The -archives option allows you to copy
<source>
-archives hdfs://host:fs_port/user/testfile.jar
</source>
+<p> User can specify a different symlink name for -archives using #. </p>
+<source>
+-archives hdfs://host:fs_port/user/testfile.tgz#tgzdir
+</source>
<p>
In this example, the input.txt file has two lines specifying the names of the two files: cachedir.jar/cache.txt and cachedir.jar/cache2.txt.
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/filecache/TaskDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/filecache/TaskDistributedCacheManager.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/filecache/TaskDistributedCacheManager.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/filecache/TaskDistributedCacheManager.java Fri Mar 4 04:22:27 2011
@@ -99,8 +99,8 @@ public class TaskDistributedCacheManager
Map<String, Path> classPaths = new HashMap<String, Path>();
if (paths != null) {
for (Path p : paths) {
- classPaths.put(p.toString(), p);
- }
+ classPaths.put(p.toUri().getPath().toString(), p);
+ }
}
for (int i = 0; i < uris.length; ++i) {
URI u = uris[i];
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java Fri Mar 4 04:22:27 2011
@@ -561,6 +561,19 @@ public class JobClient extends Configure
return newPath;
}
+ private URI getPathURI(Path destPath, String fragment)
+ throws URISyntaxException {
+ URI pathURI = destPath.toUri();
+ if (pathURI.getFragment() == null) {
+ if (fragment == null) {
+ pathURI = new URI(pathURI.toString() + "#" + destPath.getName());
+ } else {
+ pathURI = new URI(pathURI.toString() + "#" + fragment);
+ }
+ }
+ return pathURI;
+ }
+
/**
* configure the jobconf of the user with the command line options of
* -libjars, -files, -archives
@@ -623,14 +636,20 @@ public class JobClient extends Configure
FileSystem.mkdirs(fs, filesDir, mapredSysPerms);
String[] fileArr = files.split(",");
for (String tmpFile: fileArr) {
- Path tmp = new Path(tmpFile);
+ URI tmpURI;
+ try {
+ tmpURI = new URI(tmpFile);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ Path tmp = new Path(tmpURI);
Path newPath = copyRemoteFiles(fs,filesDir, tmp, job, replication);
try {
- URI pathURI = new URI(newPath.toUri().toString() + "#" + newPath.getName());
+ URI pathURI = getPathURI(newPath, tmpURI.getFragment());
DistributedCache.addCacheFile(pathURI, job);
} catch(URISyntaxException ue) {
//should not throw a uri exception
- throw new IOException("Failed to create uri for " + tmpFile);
+ throw new IOException("Failed to create uri for " + tmpFile, ue);
}
DistributedCache.createSymlink(job);
}
@@ -652,14 +671,20 @@ public class JobClient extends Configure
FileSystem.mkdirs(fs, archivesDir, mapredSysPerms);
String[] archivesArr = archives.split(",");
for (String tmpArchives: archivesArr) {
- Path tmp = new Path(tmpArchives);
+ URI tmpURI;
+ try {
+ tmpURI = new URI(tmpArchives);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
+ }
+ Path tmp = new Path(tmpURI);
Path newPath = copyRemoteFiles(fs, archivesDir, tmp, job, replication);
try {
- URI pathURI = new URI(newPath.toUri().toString() + "#" + newPath.getName());
+ URI pathURI = getPathURI(newPath, tmpURI.getFragment());
DistributedCache.addCacheArchive(pathURI, job);
} catch(URISyntaxException ue) {
//should not throw an uri excpetion
- throw new IOException("Failed to create uri for " + tmpArchives);
+ throw new IOException("Failed to create uri for " + tmpArchives, ue);
}
DistributedCache.createSymlink(job);
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestPath.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestPath.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestPath.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestPath.java Fri Mar 4 04:22:27 2011
@@ -18,7 +18,12 @@
package org.apache.hadoop.fs;
-import java.util.*;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+
import junit.framework.TestCase;
public class TestPath extends TestCase {
@@ -28,6 +33,8 @@ public class TestPath extends TestCase {
toStringTest("/foo/bar");
toStringTest("foo");
toStringTest("foo/bar");
+ toStringTest("/foo/bar#boo");
+ toStringTest("foo/bar#boo");
boolean emptyException = false;
try {
toStringTest("");
@@ -43,6 +50,8 @@ public class TestPath extends TestCase {
toStringTest("c:foo/bar");
toStringTest("c:foo/bar");
toStringTest("c:/foo/bar");
+ toStringTest("C:/foo/bar#boo");
+ toStringTest("C:foo/bar#boo");
}
}
@@ -148,5 +157,25 @@ public class TestPath extends TestCase {
assertEquals("foo://bar/baz", new Path("foo://bar/","/baz").toString());
}
+ public void testURI() throws URISyntaxException, IOException {
+ URI uri = new URI("file:///bar#baz");
+ Path path = new Path(uri);
+ assertTrue(uri.equals(new URI(path.toString())));
+
+ FileSystem fs = path.getFileSystem(new Configuration());
+ assertTrue(uri.equals(new URI(fs.makeQualified(path).toString())));
+
+ // uri without hash
+ URI uri2 = new URI("file:///bar/baz");
+ assertTrue(
+ uri2.equals(new URI(fs.makeQualified(new Path(uri2)).toString())));
+ assertEquals("foo://bar/baz#boo", new Path("foo://bar/", new Path(new URI(
+ "/baz#boo"))).toString());
+ assertEquals("foo://bar/baz/fud#boo", new Path(new Path(new URI(
+ "foo://bar/baz#bud")), new Path(new URI("fud#boo"))).toString());
+ // if the child uri is absolute path
+ assertEquals("foo://bar/fud#boo", new Path(new Path(new URI(
+ "foo://bar/baz#bud")), new Path(new URI("/fud#boo"))).toString());
+ }
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java Fri Mar 4 04:22:27 2011
@@ -18,12 +18,14 @@ package org.apache.hadoop.mapred;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.IOException;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -60,15 +62,57 @@ public class TestCommandLineJobSubmissio
FileOutputStream fstream = new FileOutputStream(f);
fstream.write("somestrings".getBytes());
fstream.close();
- String[] args = new String[6];
+ File f1 = new File(thisbuildDir, "files_tmp1");
+ fstream = new FileOutputStream(f1);
+ fstream.write("somestrings".getBytes());
+ fstream.close();
+
+ // copy files to dfs
+ Path cachePath = new Path("/cacheDir");
+ if (!fs.mkdirs(cachePath)) {
+ throw new IOException(
+ "Mkdirs failed to create " + cachePath.toString());
+ }
+ Path localCachePath = new Path(System.getProperty("test.cache.data"));
+ Path txtPath = new Path(localCachePath, new Path("test.txt"));
+ Path jarPath = new Path(localCachePath, new Path("test.jar"));
+ Path zipPath = new Path(localCachePath, new Path("test.zip"));
+ Path tarPath = new Path(localCachePath, new Path("test.tar"));
+ Path tgzPath = new Path(localCachePath, new Path("test.tgz"));
+ fs.copyFromLocalFile(txtPath, cachePath);
+ fs.copyFromLocalFile(jarPath, cachePath);
+ fs.copyFromLocalFile(zipPath, cachePath);
+
+ // construct options for -files
+ String[] files = new String[3];
+ files[0] = f.toString();
+ files[1] = f1.toString() + "#localfilelink";
+ files[2] =
+ fs.getUri().resolve(cachePath + "/test.txt#dfsfilelink").toString();
+
+ // construct options for -libjars
+ String[] libjars = new String[2];
+ libjars[0] = "build/test/testjar/testjob.jar";
+ libjars[1] = fs.getUri().resolve(cachePath + "/test.jar").toString();
+
+ // construct options for -archives
+ String[] archives = new String[3];
+ archives[0] = tgzPath.toString();
+ archives[1] = tarPath + "#tarlink";
+ archives[2] =
+ fs.getUri().resolve(cachePath + "/test.zip#ziplink").toString();
+
+ String[] args = new String[8];
args[0] = "-files";
- args[1] = f.toString();
+ args[1] = StringUtils.arrayToString(files);
args[2] = "-libjars";
// the testjob.jar as a temporary jar file
// rather than creating its own
- args[3] = "build/test/testjar/testjob.jar";
- args[4] = input.toString();
- args[5] = output.toString();
+ args[3] = StringUtils.arrayToString(libjars);
+ args[4] = "-archives";
+ args[5] = StringUtils.arrayToString(archives);
+ args[6] = input.toString();
+ args[7] = output.toString();
JobConf jobConf = mr.createJobConf();
//before running the job, verify that libjar is not in client classpath
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestGenericOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestGenericOptionsParser.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestGenericOptionsParser.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestGenericOptionsParser.java Fri Mar 4 04:22:27 2011
@@ -87,4 +87,48 @@ public class TestGenericOptionsParser ex
localFs.delete(new Path(testDir.getAbsolutePath()), true);
}
+
+ public void testFilesOption() throws Exception {
+ File tmpFile = new File(testDir, "tmpfile");
+ Path tmpPath = new Path(tmpFile.toString());
+ localFs.create(tmpPath);
+ String[] args = new String[2];
+ // pass a files option
+ args[0] = "-files";
+ args[1] = tmpFile.toString();
+ new GenericOptionsParser(conf, args);
+ String files = conf.get("tmpfiles");
+ assertNotNull("files is null", files);
+ assertEquals("files option does not match",
+ localFs.makeQualified(tmpPath).toString(), files);
+
+ // pass file as uri
+ Configuration conf1 = new Configuration();
+ URI tmpURI = new URI(tmpFile.toString() + "#link");
+ args[0] = "-files";
+ args[1] = tmpURI.toString();
+ new GenericOptionsParser(conf1, args);
+ files = conf1.get("tmpfiles");
+ assertNotNull("files is null", files);
+ assertEquals("files option does not match",
+ localFs.makeQualified(new Path(tmpURI)).toString(), files);
+
+ // pass a file that does not exist.
+ // GenericOptionParser should throw exception
+ Configuration conf2 = new Configuration();
+ args[0] = "-files";
+ args[1] = "file:///xyz.txt";
+ Throwable th = null;
+ try {
+ new GenericOptionsParser(conf2, args);
+ } catch (Exception e) {
+ th = e;
+ }
+ assertNotNull("throwable is null", th);
+ assertTrue("FileNotFoundException is not thrown",
+ th instanceof FileNotFoundException);
+ files = conf2.get("tmpfiles");
+ assertNull("files is not null", files);
+ }
+
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/testshell/ExternalMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/testshell/ExternalMapReduce.java?rev=1077512&r1=1077511&r2=1077512&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/testshell/ExternalMapReduce.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/testshell/ExternalMapReduce.java Fri Mar 4 04:22:27 2011
@@ -66,12 +66,21 @@ public class ExternalMapReduce extends C
if (classpath.indexOf("testjob.jar") == -1) {
throw new IOException("failed to find in the library " + classpath);
}
+ if (classpath.indexOf("test.jar") == -1) {
+ throw new IOException("failed to find the library test.jar in"
+ + classpath);
+ }
//fork off ls to see if the file exists.
// java file.exists() will not work on
// cygwin since it is a symlink
- String[] argv = new String[2];
+ String[] argv = new String[7];
argv[0] = "ls";
argv[1] = "files_tmp";
+ argv[2] = "localfilelink";
+ argv[3] = "dfsfilelink";
+ argv[4] = "tarlink";
+ argv[5] = "ziplink";
+ argv[6] = "test.tgz";
Process p = Runtime.getRuntime().exec(argv);
int ret = -1;
try {