You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2013/04/01 18:47:27 UTC
svn commit: r1463203 [2/7] - in
/hadoop/common/branches/HDFS-347/hadoop-common-project: hadoop-annotations/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/
hadoop-common/ hadoop-common/dev-support/ hadoop-common/src/ hadoop-co...
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Mon Apr 1 16:47:16 2013
@@ -19,18 +19,29 @@
package org.apache.hadoop.fs;
import java.io.*;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.Attributes;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import org.apache.commons.collections.map.CaseInsensitiveMap;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
@@ -46,6 +57,13 @@ public class FileUtil {
private static final Log LOG = LogFactory.getLog(FileUtil.class);
+ /* The error code is defined in winutils to indicate insufficient
+ * privilege to create symbolic links. This value need to keep in
+ * sync with the constant of the same name in:
+ * "src\winutils\common.h"
+ * */
+ public static final int SYMLINK_NO_PRIVILEGE = 2;
+
/**
* convert an array of FileStatus to an array of Path
*
@@ -470,45 +488,13 @@ public class FileUtil {
}
/**
- * This class is only used on windows to invoke the cygpath command.
- */
- private static class CygPathCommand extends Shell {
- String[] command;
- String result;
- CygPathCommand(String path) throws IOException {
- command = new String[]{"cygpath", "-u", path};
- run();
- }
- String getResult() throws IOException {
- return result;
- }
- @Override
- protected String[] getExecString() {
- return command;
- }
- @Override
- protected void parseExecResult(BufferedReader lines) throws IOException {
- String line = lines.readLine();
- if (line == null) {
- throw new IOException("Can't convert '" + command[2] +
- " to a cygwin path");
- }
- result = line;
- }
- }
-
- /**
* Convert a os-native filename to a path that works for the shell.
* @param filename The filename to convert
* @return The unix pathname
* @throws IOException on windows, there can be problems with the subprocess
*/
public static String makeShellPath(String filename) throws IOException {
- if (Path.WINDOWS) {
- return new CygPathCommand(filename).getResult();
- } else {
- return filename;
- }
+ return filename;
}
/**
@@ -658,7 +644,7 @@ public class FileUtil {
untarCommand.append(FileUtil.makeShellPath(untarDir));
untarCommand.append("' ; ");
untarCommand.append("tar -xf ");
-
+
if (gzipped) {
untarCommand.append(" -)");
} else {
@@ -731,7 +717,7 @@ public class FileUtil {
/**
* Class for creating hardlinks.
- * Supports Unix, Cygwin, WindXP.
+ * Supports Unix, WindXP.
* @deprecated Use {@link org.apache.hadoop.fs.HardLink}
*/
@Deprecated
@@ -743,21 +729,67 @@ public class FileUtil {
/**
* Create a soft link between a src and destination
- * only on a local disk. HDFS does not support this
+ * only on a local disk. HDFS does not support this.
+ * On Windows, when symlink creation fails due to security
+ * setting, we will log a warning. The return code in this
+ * case is 2.
* @param target the target for symlink
* @param linkname the symlink
* @return value returned by the command
*/
public static int symLink(String target, String linkname) throws IOException{
- String cmd = "ln -s " + target + " " + linkname;
- Process p = Runtime.getRuntime().exec(cmd, null);
- int returnVal = -1;
- try{
- returnVal = p.waitFor();
- } catch(InterruptedException e){
- //do nothing as of yet
+ // Run the input paths through Java's File so that they are converted to the
+ // native OS form
+ File targetFile = new File(target);
+ File linkFile = new File(linkname);
+
+ // If not on Java7+, copy a file instead of creating a symlink since
+ // Java6 has close to no support for symlinks on Windows. Specifically
+ // File#length and File#renameTo do not work as expected.
+ // (see HADOOP-9061 for additional details)
+ // We still create symlinks for directories, since the scenario in this
+ // case is different. The directory content could change in which
+ // case the symlink loses its purpose (for example task attempt log folder
+ // is symlinked under userlogs and userlogs are generated afterwards).
+ if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) {
+ try {
+ LOG.info("FileUtil#symlink: On Java6, copying file instead "
+ + linkname + " -> " + target);
+ org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile);
+ } catch (IOException ex) {
+ LOG.warn("FileUtil#symlink failed to copy the file with error: "
+ + ex.getMessage());
+ // Exit with non-zero exit code
+ return 1;
+ }
+ return 0;
}
- return returnVal;
+
+ String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(),
+ linkFile.getPath());
+ ShellCommandExecutor shExec = new ShellCommandExecutor(cmd);
+ try {
+ shExec.execute();
+ } catch (Shell.ExitCodeException ec) {
+ int returnVal = ec.getExitCode();
+ if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
+ LOG.warn("Fail to create symbolic links on Windows. "
+ + "The default security settings in Windows disallow non-elevated "
+ + "administrators and all non-administrators from creating symbolic links. "
+ + "This behavior can be changed in the Local Security Policy management console");
+ } else if (returnVal != 0) {
+ LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
+ + returnVal + " with: " + ec.getMessage());
+ }
+ return returnVal;
+ } catch (IOException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Error while create symlink " + linkname + " to " + target
+ + "." + " Exception: " + StringUtils.stringifyException(e));
+ }
+ throw e;
+ }
+ return shExec.getExitCode();
}
/**
@@ -781,30 +813,120 @@ public class FileUtil {
* @param recursive true, if permissions should be changed recursively
* @return the exit code from the command.
* @throws IOException
- * @throws InterruptedException
*/
public static int chmod(String filename, String perm, boolean recursive)
- throws IOException, InterruptedException {
- StringBuilder cmdBuf = new StringBuilder();
- cmdBuf.append("chmod ");
- if (recursive) {
- cmdBuf.append("-R ");
- }
- cmdBuf.append(perm).append(" ");
- cmdBuf.append(filename);
- String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
- ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
+ throws IOException {
+ String [] cmd = Shell.getSetPermissionCommand(perm, recursive);
+ String[] args = new String[cmd.length + 1];
+ System.arraycopy(cmd, 0, args, 0, cmd.length);
+ args[cmd.length] = new File(filename).getPath();
+ ShellCommandExecutor shExec = new ShellCommandExecutor(args);
try {
shExec.execute();
- }catch(Exception e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Error while changing permission : " + filename
- + " Exception: ", e);
+ }catch(IOException e) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Error while changing permission : " + filename
+ +" Exception: " + StringUtils.stringifyException(e));
}
}
return shExec.getExitCode();
}
+
+ /**
+ * Set the ownership on a file / directory. User name and group name
+ * cannot both be null.
+ * @param file the file to change
+ * @param username the new user owner name
+ * @param groupname the new group owner name
+ * @throws IOException
+ */
+ public static void setOwner(File file, String username,
+ String groupname) throws IOException {
+ if (username == null && groupname == null) {
+ throw new IOException("username == null && groupname == null");
+ }
+ String arg = (username == null ? "" : username)
+ + (groupname == null ? "" : ":" + groupname);
+ String [] cmd = Shell.getSetOwnerCommand(arg);
+ execCommand(file, cmd);
+ }
+
+ /**
+ * Set permissions to the required value. Uses the java primitives instead
+ * of forking if group == other.
+ * @param f the file to change
+ * @param permission the new permissions
+ * @throws IOException
+ */
+ public static void setPermission(File f, FsPermission permission
+ ) throws IOException {
+ FsAction user = permission.getUserAction();
+ FsAction group = permission.getGroupAction();
+ FsAction other = permission.getOtherAction();
+
+ // use the native/fork if the group/other permissions are different
+ // or if the native is available or on Windows
+ if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) {
+ execSetPermission(f, permission);
+ return;
+ }
+
+ boolean rv = true;
+
+ // read perms
+ rv = f.setReadable(group.implies(FsAction.READ), false);
+ checkReturnValue(rv, f, permission);
+ if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) {
+ rv = f.setReadable(user.implies(FsAction.READ), true);
+ checkReturnValue(rv, f, permission);
+ }
+
+ // write perms
+ rv = f.setWritable(group.implies(FsAction.WRITE), false);
+ checkReturnValue(rv, f, permission);
+ if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) {
+ rv = f.setWritable(user.implies(FsAction.WRITE), true);
+ checkReturnValue(rv, f, permission);
+ }
+
+ // exec perms
+ rv = f.setExecutable(group.implies(FsAction.EXECUTE), false);
+ checkReturnValue(rv, f, permission);
+ if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) {
+ rv = f.setExecutable(user.implies(FsAction.EXECUTE), true);
+ checkReturnValue(rv, f, permission);
+ }
+ }
+
+ private static void checkReturnValue(boolean rv, File p,
+ FsPermission permission
+ ) throws IOException {
+ if (!rv) {
+ throw new IOException("Failed to set permissions of path: " + p +
+ " to " +
+ String.format("%04o", permission.toShort()));
+ }
+ }
+
+ private static void execSetPermission(File f,
+ FsPermission permission
+ ) throws IOException {
+ if (NativeIO.isAvailable()) {
+ NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort());
+ } else {
+ execCommand(f, Shell.getSetPermissionCommand(
+ String.format("%04o", permission.toShort()), false));
+ }
+ }
+ static String execCommand(File f, String... cmd) throws IOException {
+ String[] args = new String[cmd.length + 1];
+ System.arraycopy(cmd, 0, args, 0, cmd.length);
+ args[cmd.length] = f.getCanonicalPath();
+ String output = Shell.execCommand(args);
+ return output;
+ }
+
/**
* Create a tmp file for a base file.
* @param basefile the base file of the tmp
@@ -892,4 +1014,97 @@ public class FileUtil {
}
return fileNames;
}
+
+ /**
+ * Create a jar file at the given path, containing a manifest with a classpath
+ * that references all specified entries.
+ *
+ * Some platforms may have an upper limit on command line length. For example,
+ * the maximum command line length on Windows is 8191 characters, but the
+ * length of the classpath may exceed this. To work around this limitation,
+ * use this method to create a small intermediate jar with a manifest that
+ * contains the full classpath. It returns the absolute path to the new jar,
+ * which the caller may set as the classpath for a new process.
+ *
+ * Environment variable evaluation is not supported within a jar manifest, so
+ * this method expands environment variables before inserting classpath entries
+ * to the manifest. The method parses environment variables according to
+ * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows,
+ * environment variables are case-insensitive. For example, %VAR% and %var%
+ * evaluate to the same value.
+ *
+ * Specifying the classpath in a jar manifest does not support wildcards, so
+ * this method expands wildcards internally. Any classpath entry that ends
+ * with * is translated to all files at that path with extension .jar or .JAR.
+ *
+ * @param inputClassPath String input classpath to bundle into the jar manifest
+ * @param pwd Path to working directory to save jar
+ * @return String absolute path to new jar
+ * @throws IOException if there is an I/O error while writing the jar file
+ */
+ public static String createJarWithClassPath(String inputClassPath, Path pwd)
+ throws IOException {
+ // Replace environment variables, case-insensitive on Windows
+ @SuppressWarnings("unchecked")
+ Map<String, String> env = Shell.WINDOWS ?
+ new CaseInsensitiveMap(System.getenv()) : System.getenv();
+ String[] classPathEntries = inputClassPath.split(File.pathSeparator);
+ for (int i = 0; i < classPathEntries.length; ++i) {
+ classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i],
+ StringUtils.ENV_VAR_PATTERN, env);
+ }
+ File workingDir = new File(pwd.toString());
+ if (!workingDir.mkdirs()) {
+ // If mkdirs returns false because the working directory already exists,
+ // then this is acceptable. If it returns false due to some other I/O
+ // error, then this method will fail later with an IOException while saving
+ // the jar.
+ LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
+ }
+
+ // Append all entries
+ List<String> classPathEntryList = new ArrayList<String>(
+ classPathEntries.length);
+ for (String classPathEntry: classPathEntries) {
+ if (classPathEntry.endsWith("*")) {
+ // Append all jars that match the wildcard
+ Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
+ FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
+ .globStatus(globPath);
+ if (wildcardJars != null) {
+ for (FileStatus wildcardJar: wildcardJars) {
+ classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
+ .toExternalForm());
+ }
+ }
+ } else {
+ // Append just this jar
+ classPathEntryList.add(new File(classPathEntry).toURI().toURL()
+ .toExternalForm());
+ }
+ }
+ String jarClassPath = StringUtils.join(" ", classPathEntryList);
+
+ // Create the manifest
+ Manifest jarManifest = new Manifest();
+ jarManifest.getMainAttributes().putValue(
+ Attributes.Name.MANIFEST_VERSION.toString(), "1.0");
+ jarManifest.getMainAttributes().putValue(
+ Attributes.Name.CLASS_PATH.toString(), jarClassPath);
+
+ // Write the manifest to output JAR file
+ File classPathJar = File.createTempFile("classpath-", ".jar", workingDir);
+ FileOutputStream fos = null;
+ BufferedOutputStream bos = null;
+ JarOutputStream jos = null;
+ try {
+ fos = new FileOutputStream(classPathJar);
+ bos = new BufferedOutputStream(fos);
+ jos = new JarOutputStream(bos, jarManifest);
+ } finally {
+ IOUtils.cleanup(LOG, jos, bos, fos);
+ }
+
+ return classPathJar.getCanonicalPath();
+ }
}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Mon Apr 1 16:47:16 2013
@@ -95,16 +95,18 @@ public class FilterFileSystem extends Fi
public URI getUri() {
return fs.getUri();
}
-
- /**
- * Returns a qualified URI whose scheme and authority identify this
- * FileSystem.
- */
+
+
@Override
protected URI getCanonicalUri() {
return fs.getCanonicalUri();
}
-
+
+ @Override
+ protected URI canonicalizeUri(URI uri) {
+ return fs.canonicalizeUri(uri);
+ }
+
/** Make sure that a path specifies a FileSystem. */
@Override
public Path makeQualified(Path path) {
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java Mon Apr 1 16:47:16 2013
@@ -25,9 +25,11 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
+import org.apache.hadoop.util.Shell;
+
/**
* Class for creating hardlinks.
- * Supports Unix/Linux, WinXP/2003/Vista via Cygwin, and Mac OS X.
+ * Supports Unix/Linux, Windows via winutils , and Mac OS X.
*
* The HardLink class was formerly a static inner class of FSUtil,
* and the methods provided were blatantly non-thread-safe.
@@ -41,7 +43,7 @@ public class HardLink {
public enum OSType {
OS_TYPE_UNIX,
- OS_TYPE_WINXP,
+ OS_TYPE_WIN,
OS_TYPE_SOLARIS,
OS_TYPE_MAC,
OS_TYPE_FREEBSD
@@ -56,7 +58,7 @@ public class HardLink {
//methods without instantiating the HardLink object
static {
osType = getOSType();
- if (osType == OSType.OS_TYPE_WINXP) {
+ if (osType == OSType.OS_TYPE_WIN) {
// Windows
getHardLinkCommand = new HardLinkCGWin();
} else {
@@ -80,14 +82,8 @@ public class HardLink {
static private OSType getOSType() {
String osName = System.getProperty("os.name");
- if (osName.contains("Windows") &&
- (osName.contains("XP")
- || osName.contains("2003")
- || osName.contains("Vista")
- || osName.contains("Windows_7")
- || osName.contains("Windows 7")
- || osName.contains("Windows7"))) {
- return OSType.OS_TYPE_WINXP;
+ if (Shell.WINDOWS) {
+ return OSType.OS_TYPE_WIN;
}
else if (osName.contains("SunOS")
|| osName.contains("Solaris")) {
@@ -258,11 +254,6 @@ public class HardLink {
/**
* Implementation of HardLinkCommandGetter class for Windows
- *
- * Note that the linkCount shell command for Windows is actually
- * a Cygwin shell command, and depends on ${cygwin}/bin
- * being in the Windows PATH environment variable, so
- * stat.exe can be found.
*/
static class HardLinkCGWin extends HardLinkCommandGetter {
//The Windows command getter impl class and its member fields are
@@ -270,14 +261,16 @@ public class HardLink {
//unit testing (sort of) on non-Win servers
static String[] hardLinkCommand = {
- "fsutil","hardlink","create", null, null};
+ Shell.WINUTILS,"hardlink","create", null, null};
static String[] hardLinkMultPrefix = {
"cmd","/q","/c","for", "%f", "in", "("};
static String hardLinkMultDir = "\\%f";
static String[] hardLinkMultSuffix = {
- ")", "do", "fsutil", "hardlink", "create", null,
+ ")", "do", Shell.WINUTILS, "hardlink", "create", null,
"%f", "1>NUL"};
- static String[] getLinkCountCommand = {"stat","-c%h", null};
+ static String[] getLinkCountCommand = {
+ Shell.WINUTILS, "hardlink",
+ "stat", null};
//Windows guarantees only 8K - 1 bytes cmd length.
//Subtract another 64b to allow for Java 'exec' overhead
static final int maxAllowedCmdArgLength = 8*1024 - 65;
@@ -328,12 +321,6 @@ public class HardLink {
String[] buf = new String[getLinkCountCommand.length];
System.arraycopy(getLinkCountCommand, 0, buf, 0,
getLinkCountCommand.length);
- //The linkCount command is actually a Cygwin shell command,
- //not a Windows shell command, so we should use "makeShellPath()"
- //instead of "getCanonicalPath()". However, that causes another
- //shell exec to "cygpath.exe", and "stat.exe" actually can handle
- //DOS-style paths (it just prints a couple hundred bytes of warning
- //to stderr), so we use the more efficient "getCanonicalPath()".
buf[getLinkCountCommand.length - 1] = file.getCanonicalPath();
return buf;
}
@@ -354,7 +341,7 @@ public class HardLink {
//add the fixed overhead of the hardLinkMult command
//(prefix, suffix, and Dir suffix)
sum += ("cmd.exe /q /c for %f in ( ) do "
- + "fsutil hardlink create \\%f %f 1>NUL ").length();
+ + Shell.WINUTILS + " hardlink create \\%f %f 1>NUL ").length();
return sum;
}
@@ -581,14 +568,10 @@ public class HardLink {
/* Create an IOException for failing to get link count. */
private static IOException createIOException(File f, String message,
String error, int exitvalue, Exception cause) {
-
- final String winErrMsg = "; Windows errors in getLinkCount are often due "
- + "to Cygwin misconfiguration";
final String s = "Failed to get link count on file " + f
+ ": message=" + message
+ "; error=" + error
- + ((osType == OSType.OS_TYPE_WINXP) ? winErrMsg : "")
+ "; exit value=" + exitvalue;
return (cause == null) ? new IOException(s) : new IOException(s, cause);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java Mon Apr 1 16:47:16 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.regex.Pattern;
import org.apache.avro.reflect.Stringable;
import org.apache.commons.lang.StringUtils;
@@ -43,9 +44,17 @@ public class Path implements Comparable
public static final String CUR_DIR = ".";
- static final boolean WINDOWS
+ public static final boolean WINDOWS
= System.getProperty("os.name").startsWith("Windows");
+ /**
+ * Pre-compiled regular expressions to detect path formats.
+ */
+ private static final Pattern hasUriScheme =
+ Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+ private static final Pattern hasDriveLetterSpecifier =
+ Pattern.compile("^/?[a-zA-Z]:");
+
private URI uri; // a hierarchical uri
/** Resolve a child path against a parent path. */
@@ -81,7 +90,7 @@ public class Path implements Comparable
resolved.getPath(), resolved.getFragment());
}
- private void checkPathArg( String path ) {
+ private void checkPathArg( String path ) throws IllegalArgumentException {
// disallow construction of a Path from an empty string
if ( path == null ) {
throw new IllegalArgumentException(
@@ -95,15 +104,16 @@ public class Path implements Comparable
/** Construct a path from a String. Path strings are URIs, but with
* unescaped elements and some additional normalization. */
- public Path(String pathString) {
+ public Path(String pathString) throws IllegalArgumentException {
checkPathArg( pathString );
// We can't use 'new URI(String)' directly, since it assumes things are
// escaped, which we don't require of Paths.
// add a slash in front of paths with Windows drive letters
- if (hasWindowsDrive(pathString, false))
- pathString = "/"+pathString;
+ if (hasWindowsDrive(pathString) && pathString.charAt(0) != '/') {
+ pathString = "/" + pathString;
+ }
// parse uri components
String scheme = null;
@@ -151,22 +161,54 @@ public class Path implements Comparable
private void initialize(String scheme, String authority, String path,
String fragment) {
try {
- this.uri = new URI(scheme, authority, normalizePath(path), null, fragment)
+ this.uri = new URI(scheme, authority, normalizePath(scheme, path), null, fragment)
.normalize();
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
}
- private String normalizePath(String path) {
- // remove double slashes & backslashes
+ /**
+ * Merge 2 paths such that the second path is appended relative to the first.
+ * The returned path has the scheme and authority of the first path. On
+ * Windows, the drive specification in the second path is discarded.
+ *
+ * @param path1 Path first path
+ * @param path2 Path second path, to be appended relative to path1
+ * @return Path merged path
+ */
+ public static Path mergePaths(Path path1, Path path2) {
+ String path2Str = path2.toUri().getPath();
+ if(hasWindowsDrive(path2Str)) {
+ path2Str = path2Str.substring(path2Str.indexOf(':')+1);
+ }
+ return new Path(path1 + path2Str);
+ }
+
+ /**
+ * Normalize a path string to use non-duplicated forward slashes as
+ * the path separator and remove any trailing path separators.
+ * @param scheme Supplies the URI scheme. Used to deduce whether we
+ * should replace backslashes or not.
+ * @param path Supplies the scheme-specific part
+ * @return Normalized path string.
+ */
+ private static String normalizePath(String scheme, String path) {
+ // Remove double forward slashes.
path = StringUtils.replace(path, "//", "/");
- if (Path.WINDOWS) {
+
+ // Remove backslashes if this looks like a Windows path. Avoid
+ // the substitution if it looks like a non-local URI.
+ if (WINDOWS &&
+ (hasWindowsDrive(path) ||
+ (scheme == null) ||
+ (scheme.isEmpty()) ||
+ (scheme.equals("file")))) {
path = StringUtils.replace(path, "\\", "/");
}
// trim trailing slash from non-root path (ignoring windows drive)
- int minLength = hasWindowsDrive(path, true) ? 4 : 1;
+ int minLength = hasWindowsDrive(path) ? 4 : 1;
if (path.length() > minLength && path.endsWith("/")) {
path = path.substring(0, path.length()-1);
}
@@ -174,17 +216,29 @@ public class Path implements Comparable
return path;
}
- private boolean hasWindowsDrive(String path, boolean slashed) {
- if (!WINDOWS) return false;
- int start = slashed ? 1 : 0;
- return
- path.length() >= start+2 &&
- (slashed ? path.charAt(0) == '/' : true) &&
- path.charAt(start+1) == ':' &&
- ((path.charAt(start) >= 'A' && path.charAt(start) <= 'Z') ||
- (path.charAt(start) >= 'a' && path.charAt(start) <= 'z'));
+ private static boolean hasWindowsDrive(String path) {
+ return (WINDOWS && hasDriveLetterSpecifier.matcher(path).find());
}
+ /**
+ * Determine whether a given path string represents an absolute path on
+ * Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+ *
+ * @param pathString Supplies the path string to evaluate.
+ * @param slashed true if the given path is prefixed with "/".
+ * @return true if the supplied path looks like an absolute path with a Windows
+ * drive-specifier.
+ */
+ public static boolean isWindowsAbsolutePath(final String pathString,
+ final boolean slashed) {
+ int start = (slashed ? 1 : 0);
+
+ return
+ hasWindowsDrive(pathString) &&
+ pathString.length() >= (start + 3) &&
+ ((pathString.charAt(start + 2) == SEPARATOR_CHAR) ||
+ (pathString.charAt(start + 2) == '\\'));
+ }
/** Convert this to a URI. */
public URI toUri() { return uri; }
@@ -202,12 +256,16 @@ public class Path implements Comparable
return (isUriPathAbsolute() &&
uri.getScheme() == null && uri.getAuthority() == null);
}
+
+ public boolean hasSchemeAndNoAuthority() {
+ return uri.getScheme() != null && uri.getAuthority() == null;
+ }
/**
* True if the path component (i.e. directory) of this URI is absolute.
*/
public boolean isUriPathAbsolute() {
- int start = hasWindowsDrive(uri.getPath(), true) ? 3 : 0;
+ int start = hasWindowsDrive(uri.getPath()) ? 3 : 0;
return uri.getPath().startsWith(SEPARATOR, start);
}
@@ -241,7 +299,7 @@ public class Path implements Comparable
public Path getParent() {
String path = uri.getPath();
int lastSlash = path.lastIndexOf('/');
- int start = hasWindowsDrive(path, true) ? 3 : 0;
+ int start = hasWindowsDrive(path) ? 3 : 0;
if ((path.length() == start) || // empty path
(lastSlash == start && path.length() == start+1)) { // at root
return null;
@@ -250,7 +308,7 @@ public class Path implements Comparable
if (lastSlash==-1) {
parent = CUR_DIR;
} else {
- int end = hasWindowsDrive(path, true) ? 3 : 0;
+ int end = hasWindowsDrive(path) ? 3 : 0;
parent = path.substring(0, lastSlash==end?end+1:lastSlash);
}
return new Path(uri.getScheme(), uri.getAuthority(), parent);
@@ -277,7 +335,7 @@ public class Path implements Comparable
if (uri.getPath() != null) {
String path = uri.getPath();
if (path.indexOf('/')==0 &&
- hasWindowsDrive(path, true) && // has windows drive
+ hasWindowsDrive(path) && // has windows drive
uri.getScheme() == null && // but no scheme
uri.getAuthority() == null) // or authority
path = path.substring(1); // remove slash before drive
@@ -364,7 +422,7 @@ public class Path implements Comparable
URI newUri = null;
try {
newUri = new URI(scheme, authority ,
- normalizePath(pathUri.getPath()), null, fragment);
+ normalizePath(scheme, pathUri.getPath()), null, fragment);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java Mon Apr 1 16:47:16 2013
@@ -43,7 +43,7 @@ public interface PositionedReadable {
throws IOException;
/**
- * Read number of bytes equalt to the length of the buffer, from a given
+ * Read number of bytes equal to the length of the buffer, from a given
* position within a file. This does not
* change the current offset of a file, and is thread-safe.
*/
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Mon Apr 1 16:47:16 2013
@@ -508,9 +508,10 @@ public class RawLocalFileSystem extends
return !super.getOwner().isEmpty();
}
- RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
+ RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
super(f.length(), f.isDirectory(), 1, defaultBlockSize,
- f.lastModified(), fs.makeQualified(new Path(f.getPath())));
+ f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(),
+ fs.getWorkingDirectory()));
}
@Override
@@ -541,9 +542,10 @@ public class RawLocalFileSystem extends
private void loadPermissionInfo() {
IOException e = null;
try {
- StringTokenizer t = new StringTokenizer(
- execCommand(new File(getPath().toUri()),
- Shell.getGET_PERMISSION_COMMAND()));
+ String output = FileUtil.execCommand(new File(getPath().toUri()),
+ Shell.getGetPermissionCommand());
+ StringTokenizer t =
+ new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX);
//expected format
//-rw------- 1 username groupname ...
String permission = t.nextToken();
@@ -552,7 +554,17 @@ public class RawLocalFileSystem extends
}
setPermission(FsPermission.valueOf(permission));
t.nextToken();
- setOwner(t.nextToken());
+
+ String owner = t.nextToken();
+ // If on windows domain, token format is DOMAIN\\user and we want to
+ // extract only the user name
+ if (Shell.WINDOWS) {
+ int i = owner.indexOf('\\');
+ if (i != -1)
+ owner = owner.substring(i + 1);
+ }
+ setOwner(owner);
+
setGroup(t.nextToken());
} catch (Shell.ExitCodeException ioe) {
if (ioe.getExitCode() != 1) {
@@ -588,17 +600,7 @@ public class RawLocalFileSystem extends
@Override
public void setOwner(Path p, String username, String groupname)
throws IOException {
- if (username == null && groupname == null) {
- throw new IOException("username == null && groupname == null");
- }
-
- if (username == null) {
- execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname);
- } else {
- //OWNER[:[GROUP]]
- String s = username + (groupname == null? "": ":" + groupname);
- execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s);
- }
+ FileUtil.setOwner(pathToFile(p), username, groupname);
}
/**
@@ -608,20 +610,12 @@ public class RawLocalFileSystem extends
public void setPermission(Path p, FsPermission permission)
throws IOException {
if (NativeIO.isAvailable()) {
- NativeIO.chmod(pathToFile(p).getCanonicalPath(),
+ NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
permission.toShort());
} else {
- execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
- String.format("%05o", permission.toShort()));
+ String perm = String.format("%04o", permission.toShort());
+ Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
+ FileUtil.makeShellPath(pathToFile(p), true)));
}
}
-
- private static String execCommand(File f, String... cmd) throws IOException {
- String[] args = new String[cmd.length + 1];
- System.arraycopy(cmd, 0, args, 0, cmd.length);
- args[cmd.length] = FileUtil.makeShellPath(f, true);
- String output = Shell.execCommand(args);
- return output;
- }
-
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java Mon Apr 1 16:47:16 2013
@@ -79,19 +79,17 @@ public abstract class TrashPolicy extend
/**
* Get an instance of the configured TrashPolicy based on the value
- * of the configuration paramater fs.trash.classname.
+ * of the configuration parameter fs.trash.classname.
*
* @param conf the configuration to be used
* @param fs the file system to be used
* @param home the home directory
* @return an instance of TrashPolicy
*/
- public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home)
- throws IOException {
- Class<? extends TrashPolicy> trashClass = conf.getClass("fs.trash.classname",
- TrashPolicyDefault.class,
- TrashPolicy.class);
- TrashPolicy trash = (TrashPolicy) ReflectionUtils.newInstance(trashClass, conf);
+ public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) {
+ Class<? extends TrashPolicy> trashClass = conf.getClass(
+ "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
+ TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf);
trash.initialize(conf, fs, home); // initialize TrashPolicy
return trash;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java Mon Apr 1 16:47:16 2013
@@ -92,7 +92,7 @@ public class TrashPolicyDefault extends
}
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
- return new Path(basePath + rmFilePath.toUri().getPath());
+ return Path.mergePaths(basePath, rmFilePath);
}
@Override
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java Mon Apr 1 16:47:16 2013
@@ -89,11 +89,9 @@ public class RawLocalFs extends Delegate
}
// NB: Use createSymbolicLink in java.nio.file.Path once available
try {
- Shell.execCommand(Shell.LINK_COMMAND, "-s",
- new URI(target.toString()).getPath(),
- new URI(link.toString()).getPath());
- } catch (URISyntaxException x) {
- throw new IOException("Invalid symlink path: "+x.getMessage());
+ Shell.execCommand(Shell.getSymlinkCommand(
+ getPathWithoutSchemeAndAuthority(target),
+ getPathWithoutSchemeAndAuthority(link)));
} catch (IOException x) {
throw new IOException("Unable to create symlink: "+x.getMessage());
}
@@ -176,4 +174,13 @@ public class RawLocalFs extends Delegate
*/
throw new AssertionError();
}
+
+ private static String getPathWithoutSchemeAndAuthority(Path path) {
+ // This code depends on Path.toString() to remove the leading slash before
+ // the drive specification on Windows.
+ Path newPath = path.isUriPathAbsolute() ?
+ new Path(null, null, path.toUri().getPath()) :
+ path;
+ return newPath.toString();
+ }
}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java Mon Apr 1 16:47:16 2013
@@ -137,9 +137,15 @@ class Jets3tFileSystemStore implements F
@Override
public boolean inodeExists(Path path) throws IOException {
- InputStream in = get(pathToKey(path), true);
+ String key = pathToKey(path);
+ InputStream in = get(key, true);
if (in == null) {
- return false;
+ if (isRoot(key)) {
+ storeINode(path, INode.DIRECTORY_INODE);
+ return true;
+ } else {
+ return false;
+ }
}
in.close();
return true;
@@ -211,7 +217,13 @@ class Jets3tFileSystemStore implements F
@Override
public INode retrieveINode(Path path) throws IOException {
- return INode.deserialize(get(pathToKey(path), true));
+ String key = pathToKey(path);
+ InputStream in = get(key, true);
+ if (in == null && isRoot(key)) {
+ storeINode(path, INode.DIRECTORY_INODE);
+ return INode.DIRECTORY_INODE;
+ }
+ return INode.deserialize(in);
}
@Override
@@ -366,6 +378,10 @@ class Jets3tFileSystemStore implements F
return blockToKey(block.getId());
}
+ private boolean isRoot(String key) {
+ return key.isEmpty() || key.equals("/");
+ }
+
@Override
public void purge() throws IOException {
try {
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java Mon Apr 1 16:47:16 2013
@@ -252,32 +252,125 @@ public class S3FileSystem extends FileSy
@Override
public boolean rename(Path src, Path dst) throws IOException {
Path absoluteSrc = makeAbsolute(src);
+ final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
INode srcINode = store.retrieveINode(absoluteSrc);
+ boolean debugEnabled = LOG.isDebugEnabled();
if (srcINode == null) {
// src path doesn't exist
+ if (debugEnabled) {
+ LOG.debug(debugPreamble + "returning false as src does not exist");
+ }
return false;
}
+
Path absoluteDst = makeAbsolute(dst);
- INode dstINode = store.retrieveINode(absoluteDst);
- if (dstINode != null && dstINode.isDirectory()) {
- absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
- dstINode = store.retrieveINode(absoluteDst);
- }
- if (dstINode != null) {
- // dst path already exists - can't overwrite
- return false;
- }
+
+ //validate the parent dir of the destination
Path dstParent = absoluteDst.getParent();
if (dstParent != null) {
+ //if the dst parent is not root, make sure it exists
INode dstParentINode = store.retrieveINode(dstParent);
- if (dstParentINode == null || dstParentINode.isFile()) {
- // dst parent doesn't exist or is a file
+ if (dstParentINode == null) {
+ // dst parent doesn't exist
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "returning false as dst parent does not exist");
+ }
+ return false;
+ }
+ if (dstParentINode.isFile()) {
+ // dst parent exists but is a file
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "returning false as dst parent exists and is a file");
+ }
+ return false;
+ }
+ }
+
+ //get status of source
+ boolean srcIsFile = srcINode.isFile();
+
+ INode dstINode = store.retrieveINode(absoluteDst);
+ boolean destExists = dstINode != null;
+ boolean destIsDir = destExists && !dstINode.isFile();
+ if (srcIsFile) {
+
+ //source is a simple file
+ if (destExists) {
+ if (destIsDir) {
+ //outcome #1 dest exists and is dir -filename to subdir of dest
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying src file under dest dir to " + absoluteDst);
+ }
+ absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+ } else {
+ //outcome #2 dest it's a file: fail iff different from src
+ boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst);
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying file onto file, outcome is " + renamingOnToSelf);
+ }
+ return renamingOnToSelf;
+ }
+ } else {
+ // #3 dest does not exist: use dest as path for rename
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying file onto file");
+ }
+ }
+ } else {
+ //here the source exists and is a directory
+ // outcomes (given we know the parent dir exists if we get this far)
+ // #1 destination is a file: fail
+ // #2 destination is a directory: create a new dir under that one
+ // #3 destination doesn't exist: create a new dir with that name
+ // #3 and #4 are only allowed if the dest path is not == or under src
+
+ if (destExists) {
+ if (!destIsDir) {
+ // #1 destination is a file: fail
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "returning false as src is a directory, but not dest");
+ }
+ return false;
+ } else {
+ // the destination dir exists
+ // destination for rename becomes a subdir of the target name
+ absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "copying src dir under dest dir to " + absoluteDst);
+ }
+ }
+ }
+ //the final destination directory is now know, so validate it for
+ //illegal moves
+
+ if (absoluteSrc.equals(absoluteDst)) {
+ //you can't rename a directory onto itself
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "Dest==source && isDir -failing");
+ }
+ return false;
+ }
+ if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) {
+ //you can't move a directory under itself
+ if (debugEnabled) {
+ LOG.debug(debugPreamble +
+ "dst is equal to or under src dir -failing");
+ }
return false;
}
}
+ //here the dest path is set up -so rename
return renameRecursive(absoluteSrc, absoluteDst);
}
-
+
private boolean renameRecursive(Path src, Path dst) throws IOException {
INode srcINode = store.retrieveINode(src);
store.storeINode(dst, srcINode);
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Mon Apr 1 16:47:16 2013
@@ -582,35 +582,58 @@ public class NativeS3FileSystem extends
public boolean rename(Path src, Path dst) throws IOException {
String srcKey = pathToKey(makeAbsolute(src));
+ final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
if (srcKey.length() == 0) {
// Cannot rename root of file system
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble +
+ "returning false as cannot rename the root of a filesystem");
+ }
return false;
}
- final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
-
+ //get status of source
+ boolean srcIsFile;
+ try {
+ srcIsFile = getFileStatus(src).isFile();
+ } catch (FileNotFoundException e) {
+ //bail out fast if the source does not exist
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "returning false as src does not exist");
+ }
+ return false;
+ }
// Figure out the final destination
- String dstKey;
+ String dstKey = pathToKey(makeAbsolute(dst));
+
try {
boolean dstIsFile = getFileStatus(dst).isFile();
if (dstIsFile) {
+ //destination is a file.
+ //you can't copy a file or a directory onto an existing file
+ //except for the special case of dest==src, which is a no-op
if(LOG.isDebugEnabled()) {
LOG.debug(debugPreamble +
- "returning false as dst is an already existing file");
+ "returning without rename as dst is an already existing file");
}
- return false;
+ //exit, returning true iff the rename is onto self
+ return srcKey.equals(dstKey);
} else {
+ //destination exists and is a directory
if(LOG.isDebugEnabled()) {
LOG.debug(debugPreamble + "using dst as output directory");
}
+ //destination goes under the dst path, with the name of the
+ //source entry
dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
}
} catch (FileNotFoundException e) {
+ //destination does not exist => the source file or directory
+ //is copied over with the name of the destination
if(LOG.isDebugEnabled()) {
LOG.debug(debugPreamble + "using dst as output destination");
}
- dstKey = pathToKey(makeAbsolute(dst));
try {
if (getFileStatus(dst.getParent()).isFile()) {
if(LOG.isDebugEnabled()) {
@@ -628,16 +651,17 @@ public class NativeS3FileSystem extends
}
}
- boolean srcIsFile;
- try {
- srcIsFile = getFileStatus(src).isFile();
- } catch (FileNotFoundException e) {
- if(LOG.isDebugEnabled()) {
- LOG.debug(debugPreamble + "returning false as src does not exist");
+ //rename to self behavior follows Posix rules and is different
+ //for directories and files -the return code is driven by src type
+ if (srcKey.equals(dstKey)) {
+ //fully resolved destination key matches source: fail
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(debugPreamble + "renamingToSelf; returning true");
}
- return false;
+ return true;
}
if (srcIsFile) {
+ //source is a file; COPY then DELETE
if(LOG.isDebugEnabled()) {
LOG.debug(debugPreamble +
"src is file, so doing copy then delete in S3");
@@ -645,9 +669,19 @@ public class NativeS3FileSystem extends
store.copy(srcKey, dstKey);
store.delete(srcKey);
} else {
+ //src is a directory
if(LOG.isDebugEnabled()) {
LOG.debug(debugPreamble + "src is directory, so copying contents");
}
+ //Verify dest is not a child of the parent
+ if (dstKey.startsWith(srcKey + "/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ debugPreamble + "cannot rename a directory to a subdirectory of self");
+ }
+ return false;
+ }
+ //create the subdir under the destination
store.storeEmptyFile(dstKey + FOLDER_SUFFIX);
List<String> keysToDelete = new ArrayList<String>();
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Mon Apr 1 16:47:16 2013
@@ -18,9 +18,10 @@
package org.apache.hadoop.fs.shell;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.LinkedList;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -72,8 +73,12 @@ abstract class CommandWithDestination ex
*/
protected void getLocalDestination(LinkedList<String> args)
throws IOException {
- String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
- dst = new PathData(new File(pathString), getConf());
+ try {
+ String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast();
+ dst = new PathData(new URI(pathString), getConf());
+ } catch (URISyntaxException e) {
+ throw new IOException("unexpected URISyntaxException", e);
+ }
}
/**
@@ -218,7 +223,13 @@ abstract class CommandWithDestination ex
*/
protected void copyFileToTarget(PathData src, PathData target) throws IOException {
src.fs.setVerifyChecksum(verifyChecksum);
- copyStreamToTarget(src.fs.open(src.path), target);
+ InputStream in = null;
+ try {
+ in = src.fs.open(src.path);
+ copyStreamToTarget(in, target);
+ } finally {
+ IOUtils.closeStream(in);
+ }
}
/**
@@ -295,4 +306,4 @@ abstract class CommandWithDestination ex
processDeleteOnExit();
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Mon Apr 1 16:47:16 2013
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.shell;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.LinkedList;
import java.util.List;
@@ -60,16 +62,20 @@ class CopyCommands {
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
- cf.parse(args);
+ try {
+ CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl");
+ cf.parse(args);
- delimiter = cf.getOpt("nl") ? "\n" : null;
+ delimiter = cf.getOpt("nl") ? "\n" : null;
- dst = new PathData(new File(args.removeLast()), getConf());
- if (dst.exists && dst.stat.isDirectory()) {
- throw new PathIsDirectoryException(dst.toString());
+ dst = new PathData(new URI(args.removeLast()), getConf());
+ if (dst.exists && dst.stat.isDirectory()) {
+ throw new PathIsDirectoryException(dst.toString());
+ }
+ srcs = new LinkedList<PathData>();
+ } catch (URISyntaxException e) {
+ throw new IOException("unexpected URISyntaxException", e);
}
- srcs = new LinkedList<PathData>();
}
@Override
@@ -188,9 +194,13 @@ class CopyCommands {
// commands operating on local paths have no need for glob expansion
@Override
protected List<PathData> expandArgument(String arg) throws IOException {
- List<PathData> items = new LinkedList<PathData>();
- items.add(new PathData(new File(arg), getConf()));
- return items;
+ try {
+ List<PathData> items = new LinkedList<PathData>();
+ items.add(new PathData(new URI(arg), getConf()));
+ return items;
+ } catch (URISyntaxException e) {
+ throw new IOException("unexpected URISyntaxException", e);
+ }
}
@Override
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java Mon Apr 1 16:47:16 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsDirectoryException;
@@ -47,13 +48,14 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.util.MinimalPrettyPrinter;
/**
- * Display contents of files
+ * Display contents or checksums of files
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -62,6 +64,7 @@ class Display extends FsCommand {
public static void registerCommands(CommandFactory factory) {
factory.addClass(Cat.class, "-cat");
factory.addClass(Text.class, "-text");
+ factory.addClass(Checksum.class, "-checksum");
}
/**
@@ -161,6 +164,36 @@ class Display extends FsCommand {
return i;
}
}
+
+ public static class Checksum extends Display {
+ public static final String NAME = "checksum";
+ public static final String USAGE = "<src> ...";
+ public static final String DESCRIPTION =
+ "Dump checksum information for files that match the file\n" +
+ "pattern <src> to stdout. Note that this requires a round-trip\n" +
+ "to a datanode storing each block of the file, and thus is not\n" +
+ "efficient to run on a large number of files. The checksum of a\n" +
+ "file depends on its content, block size and the checksum\n" +
+ "algorithm and parameters used for creating the file.";
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ if (item.stat.isDirectory()) {
+ throw new PathIsDirectoryException(item.toString());
+ }
+
+ FileChecksum checksum = item.fs.getFileChecksum(item.path);
+ if (checksum == null) {
+ out.printf("%s\tNONE\t\n", item.toString());
+ } else {
+ String checksumString = StringUtils.byteToHexString(
+ checksum.getBytes(), 0, checksum.getLength());
+ out.printf("%s\t%s\t%s\n",
+ item.toString(), checksum.getAlgorithmName(),
+ checksumString);
+ }
+ }
+ }
protected class TextRecordInputStream extends InputStream {
SequenceFile.Reader r;
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java Mon Apr 1 16:47:16 2013
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
+import java.util.regex.Pattern;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -39,6 +40,9 @@ import org.apache.hadoop.fs.PathNotFound
/**
* Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
+ * PathData ensures that the returned path string will be the same as the
+ * one passed in during initialization (unlike Path objects which can
+ * modify the path string).
* The stat field will be null if the path does not exist.
*/
@InterfaceAudience.Private
@@ -51,6 +55,20 @@ public class PathData implements Compara
public FileStatus stat;
public boolean exists;
+ /* True if the URI scheme was not present in the pathString but inferred.
+ */
+ private boolean inferredSchemeFromPath = false;
+
+ /**
+ * Pre-compiled regular expressions to detect path formats.
+ */
+ private static final Pattern potentialUri =
+ Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:");
+ private static final Pattern windowsNonUriAbsolutePath1 =
+ Pattern.compile("^/?[a-zA-Z]:\\\\");
+ private static final Pattern windowsNonUriAbsolutePath2 =
+ Pattern.compile("^/?[a-zA-Z]:/");
+
/**
* Creates an object to wrap the given parameters as fields. The string
* used to create the path will be recorded since the Path object does not
@@ -67,12 +85,12 @@ public class PathData implements Compara
* Creates an object to wrap the given parameters as fields. The string
* used to create the path will be recorded since the Path object does not
* return exactly the same string used to initialize it
- * @param localPath a local File
+ * @param localPath a local URI
* @param conf the configuration file
* @throws IOException if anything goes wrong...
*/
- public PathData(File localPath, Configuration conf) throws IOException {
- this(FileSystem.getLocal(conf), localPath.toString());
+ public PathData(URI localPath, Configuration conf) throws IOException {
+ this(FileSystem.getLocal(conf), localPath.getPath());
}
/**
@@ -87,6 +105,39 @@ public class PathData implements Compara
}
/**
+ * Validates the given Windows path.
+ * Throws IOException on failure.
+ * @param pathString a String of the path suppliued by the user.
+ */
+ private void ValidateWindowsPath(String pathString)
+ throws IOException
+ {
+ if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+ // Forward slashes disallowed in a backslash-separated path.
+ if (pathString.indexOf('/') != -1) {
+ throw new IOException("Invalid path string " + pathString);
+ }
+
+ inferredSchemeFromPath = true;
+ return;
+ }
+
+ // Is it a forward slash-separated absolute path?
+ if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+ inferredSchemeFromPath = true;
+ return;
+ }
+
+ // Does it look like a URI? If so then just leave it alone.
+ if (potentialUri.matcher(pathString).find()) {
+ return;
+ }
+
+ // Looks like a relative path on Windows.
+ return;
+ }
+
+ /**
* Creates an object to wrap the given parameters as fields. The string
* used to create the path will be recorded since the Path object does not
* return exactly the same string used to initialize it.
@@ -100,6 +151,10 @@ public class PathData implements Compara
this.uri = stringToUri(pathString);
this.path = fs.makeQualified(new Path(uri));
setStat(stat);
+
+ if (Path.WINDOWS) {
+ ValidateWindowsPath(pathString);
+ }
}
// need a static method for the ctor above
@@ -236,7 +291,7 @@ public class PathData implements Compara
* Given a child of this directory, use the directory's path and the child's
* basename to construct the string to the child. This preserves relative
* paths since Path will fully qualify.
- * @param child a path contained within this directory
+ * @param childPath a path contained within this directory
* @return String of the path relative to this directory
*/
private String getStringForChildPath(Path childPath) {
@@ -283,7 +338,8 @@ public class PathData implements Compara
URI globUri = globPath.toUri();
if (globUri.getScheme() != null) {
globType = PathType.HAS_SCHEME;
- } else if (new File(globUri.getPath()).isAbsolute()) {
+ } else if (!globUri.getPath().isEmpty() &&
+ new Path(globUri.getPath()).isAbsolute()) {
globType = PathType.SCHEMELESS_ABSOLUTE;
} else {
globType = PathType.RELATIVE;
@@ -386,7 +442,14 @@ public class PathData implements Compara
// No interpretation of symbols. Just decode % escaped chars.
String decodedRemainder = uri.getSchemeSpecificPart();
- if (scheme == null) {
+ // Drop the scheme if it was inferred to ensure fidelity between
+ // the input and output path strings.
+ if ((scheme == null) || (inferredSchemeFromPath)) {
+ if (Path.isWindowsAbsolutePath(decodedRemainder, true)) {
+ // Strip the leading '/' added in stringToUri so users see a valid
+ // Windows path.
+ decodedRemainder = decodedRemainder.substring(1);
+ }
return decodedRemainder;
} else {
StringBuilder buffer = new StringBuilder();
@@ -409,13 +472,56 @@ public class PathData implements Compara
return ((LocalFileSystem)fs).pathToFile(path);
}
+ /** Normalize the given Windows path string. This does the following:
+ * 1. Adds "file:" scheme for absolute paths.
+ * 2. Ensures the scheme-specific part starts with '/' per RFC2396.
+ * 3. Replaces backslash path separators with forward slashes.
+ * @param pathString Path string supplied by the user.
+ * @return normalized absolute path string. Returns the input string
+ * if it is not a Windows absolute path.
+ */
+ private static String normalizeWindowsPath(String pathString)
+ throws IOException
+ {
+ if (!Path.WINDOWS) {
+ return pathString;
+ }
+
+ boolean slashed =
+ ((pathString.length() >= 1) && (pathString.charAt(0) == '/'));
+
+ // Is it a backslash-separated absolute path?
+ if (windowsNonUriAbsolutePath1.matcher(pathString).find()) {
+ // Forward slashes disallowed in a backslash-separated path.
+ if (pathString.indexOf('/') != -1) {
+ throw new IOException("Invalid path string " + pathString);
+ }
+
+ pathString = pathString.replace('\\', '/');
+ return "file:" + (slashed ? "" : "/") + pathString;
+ }
+
+ // Is it a forward slash-separated absolute path?
+ if (windowsNonUriAbsolutePath2.matcher(pathString).find()) {
+ return "file:" + (slashed ? "" : "/") + pathString;
+ }
+
+ // Is it a backslash-separated relative file path (no scheme and
+ // no drive-letter specifier)?
+ if ((pathString.indexOf(':') == -1) && (pathString.indexOf('\\') != -1)) {
+ pathString = pathString.replace('\\', '/');
+ }
+
+ return pathString;
+ }
+
/** Construct a URI from a String with unescaped special characters
- * that have non-standard sematics. e.g. /, ?, #. A custom parsing
- * is needed to prevent misbihaviors.
+ * that have non-standard semantics. e.g. /, ?, #. A custom parsing
+ * is needed to prevent misbehavior.
* @param pathString The input path in string form
* @return URI
*/
- private static URI stringToUri(String pathString) {
+ private static URI stringToUri(String pathString) throws IOException {
// We can't use 'new URI(String)' directly. Since it doesn't do quoting
// internally, the internal parser may fail or break the string at wrong
// places. Use of multi-argument ctors will quote those chars for us,
@@ -424,9 +530,10 @@ public class PathData implements Compara
// parse uri components
String scheme = null;
String authority = null;
-
int start = 0;
+ pathString = normalizeWindowsPath(pathString);
+
// parse uri scheme, if any
int colon = pathString.indexOf(':');
int slash = pathString.indexOf('/');
@@ -445,8 +552,7 @@ public class PathData implements Compara
authority = pathString.substring(start, authEnd);
start = authEnd;
}
-
- // uri path is the rest of the string. ? or # are not interpreated,
+ // uri path is the rest of the string. ? or # are not interpreted,
// but any occurrence of them will be quoted by the URI ctor.
String path = pathString.substring(start, pathString.length());
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Mon Apr 1 16:47:16 2013
@@ -61,6 +61,7 @@ import org.apache.hadoop.security.authen
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Shell;
import org.mortbay.io.Buffer;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Handler;
@@ -305,6 +306,13 @@ public class HttpServer implements Filte
ret.setAcceptQueueSize(128);
ret.setResolveNames(false);
ret.setUseDirectBuffers(false);
+ if(Shell.WINDOWS) {
+ // result of setting the SO_REUSEADDR flag is different on Windows
+ // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
+ // without this 2 NN's can start on the same machine and listen on
+ // the same port with indeterminate routing of incoming requests to them
+ ret.setReuseAddress(false);
+ }
ret.setHeaderBufferSize(1024*64);
return ret;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java Mon Apr 1 16:47:16 2013
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.
/**
* A byte sequence that is usable as a key or value.
- * It is resizable and distinguishes between the size of the seqeunce and
+ * It is resizable and distinguishes between the size of the sequence and
* the current capacity. The hash function is the front of the md5 of the
* buffer. The sort order is the same as memcmp.
*/
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java Mon Apr 1 16:47:16 2013
@@ -203,8 +203,8 @@ public class ReadaheadPool {
// It's also possible that we'll end up requesting readahead on some
// other FD, which may be wasted work, but won't cause a problem.
try {
- NativeIO.posixFadviseIfPossible(fd, off, len,
- NativeIO.POSIX_FADV_WILLNEED);
+ NativeIO.POSIX.posixFadviseIfPossible(fd, off, len,
+ NativeIO.POSIX.POSIX_FADV_WILLNEED);
} catch (IOException ioe) {
if (canceled) {
// no big deal - the reader canceled the request and closed
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java Mon Apr 1 16:47:16 2013
@@ -22,6 +22,7 @@ import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -30,7 +31,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.io.nativeio.Errno;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIOException;
-import org.apache.hadoop.io.nativeio.NativeIO.Stat;
+import org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat;
import org.apache.hadoop.security.UserGroupInformation;
/**
@@ -120,7 +121,7 @@ public class SecureIOUtils {
FileInputStream fis = new FileInputStream(f);
boolean success = false;
try {
- Stat stat = NativeIO.getFstat(fis.getFD());
+ Stat stat = NativeIO.POSIX.getFstat(fis.getFD());
checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
expectedGroup);
success = true;
@@ -166,35 +167,30 @@ public class SecureIOUtils {
if (skipSecurity) {
return insecureCreateForWrite(f, permissions);
} else {
- // Use the native wrapper around open(2)
- try {
- FileDescriptor fd = NativeIO.open(f.getAbsolutePath(),
- NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL,
- permissions);
- return new FileOutputStream(fd);
- } catch (NativeIOException nioe) {
- if (nioe.getErrno() == Errno.EEXIST) {
- throw new AlreadyExistsException(nioe);
- }
- throw nioe;
- }
+ return NativeIO.getCreateForWriteFileOutputStream(f, permissions);
}
}
private static void checkStat(File f, String owner, String group,
String expectedOwner,
String expectedGroup) throws IOException {
+ boolean success = true;
if (expectedOwner != null &&
!expectedOwner.equals(owner)) {
- throw new IOException(
- "Owner '" + owner + "' for path " + f + " did not match " +
- "expected owner '" + expectedOwner + "'");
+ if (Path.WINDOWS) {
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(expectedOwner);
+ final String adminsGroupString = "Administrators";
+ success = owner.equals(adminsGroupString)
+ && Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString);
+ } else {
+ success = false;
+ }
}
- if (expectedGroup != null &&
- !expectedGroup.equals(group)) {
+ if (!success) {
throw new IOException(
- "Group '" + group + "' for path " + f + " did not match " +
- "expected group '" + expectedGroup + "'");
+ "Owner '" + owner + "' for path " + f + " did not match " +
+ "expected owner '" + expectedOwner + "'");
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1463203&r1=1463202&r2=1463203&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java Mon Apr 1 16:47:16 2013
@@ -128,7 +128,7 @@ public class Text extends BinaryComparab
/**
* Returns the Unicode Scalar Value (32-bit integer value)
* for the character at <code>position</code>. Note that this
- * method avoids using the converter or doing String instatiation
+ * method avoids using the converter or doing String instantiation
* @return the Unicode scalar value at position or -1
* if the position is invalid or points to a
* trailing byte
@@ -527,7 +527,7 @@ public class Text extends BinaryComparab
int length = 0;
int state = LEAD_BYTE;
while (count < start+len) {
- int aByte = ((int) utf8[count] & 0xFF);
+ int aByte = utf8[count] & 0xFF;
switch (state) {
case LEAD_BYTE: