You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2013/03/01 20:37:06 UTC
svn commit: r1451695 - in
/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common:
./ src/ src/main/bin/ src/main/docs/ src/main/java/
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/io/
src/main/java/org/apache...
Author: suresh
Date: Fri Mar 1 19:37:03 2013
New Revision: 1451695
URL: http://svn.apache.org/r1451695
Log:
Merge trunk to branch-trunk-win
Added:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
- copied unchanged from r1451693, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java
- copied unchanged from r1451693, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt Fri Mar 1 19:37:03 2013
@@ -355,6 +355,11 @@ Release 2.0.4-beta - UNRELEASED
HADOOP-9117. replace protoc ant plugin exec with a maven plugin. (tucu)
+ HADOOP-9279. Document the need to build hadoop-maven-plugins for
+ eclipse and separate project builds. (Tsuyoshi Ozawa via suresh)
+
+ HADOOP-9334. Upgrade netty version. (Nicolas Liochon via suresh)
+
OPTIMIZATIONS
BUG FIXES
@@ -376,6 +381,28 @@ Release 2.0.4-beta - UNRELEASED
HADOOP-9304. remove addition of avro genreated-sources dirs to build. (tucu)
+ HADOOP-9267. hadoop -help, -h, --help should show usage instructions.
+ (Andrew Wang via atm)
+
+ HADOOP-8569. CMakeLists.txt: define _GNU_SOURCE and _LARGEFILE_SOURCE.
+ (Colin Patrick McCabe via atm)
+
+ HADOOP-9323. Fix typos in API documentation. (suresh)
+
+ HADOOP-7487. DF should throw a more reasonable exception when mount cannot
+ be determined. (Andrew Wang via atm)
+
+ HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern.
+ (Arpit Gupta via suresh)
+
+ HADOOP-9342. Remove jline from distribution. (thw via tucu)
+
+ HADOOP-9230. TestUniformSizeInputFormat fails intermittently.
+ (kkambatl via tucu)
+
+ HADOOP-9349. Confusing output when running hadoop version from one hadoop
+ installation when HADOOP_HOME points to another. (sandyr via tucu)
+
Release 2.0.3-alpha - 2013-02-06
INCOMPATIBLE CHANGES
@@ -1351,7 +1378,10 @@ Release 0.23.7 - UNRELEASED
permissions (Ivan A. Veselovsky via bobby)
HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure
- (Ivan A. Veselovsky via bobby)
+ (Ivan A. Veselovsky via bobby)
+
+ HADOOP-9336. Allow UGI of current connection to be queried. (Daryn Sharp
+ via kihwal)
OPTIMIZATIONS
@@ -1363,6 +1393,9 @@ Release 0.23.7 - UNRELEASED
HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage
option (Andy Isaacson via tgraves)
+ HADOOP-9339. IPC.Server incorrectly sets UGI auth type (Daryn Sharp via
+ kihwal)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1448457-1451693
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml Fri Mar 1 19:37:03 2013
@@ -220,6 +220,10 @@
<version>3.4.2</version>
<exclusions>
<exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <exclusion>
<!-- otherwise seems to drag in junit 3.8.1 via jline -->
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt Fri Mar 1 19:37:03 2013
@@ -90,7 +90,8 @@ find_package(ZLIB REQUIRED)
SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
set(D main/native/src/org/apache/hadoop)
set(T main/native/src/test/org/apache/hadoop)
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop Fri Mar 1 19:37:03 2013
@@ -50,6 +50,12 @@ fi
COMMAND=$1
case $COMMAND in
+ # usage flags
+ --help|-help|-h)
+ print_usage
+ exit
+ ;;
+
#hdfs commands
namenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|fetchdt|oiv|dfsgroups)
echo "DEPRECATED: Use of this script to execute hdfs command is deprecated." 1>&2
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1448457-1451693
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1448457-1451693
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java Fri Mar 1 19:37:03 2013
@@ -21,8 +21,6 @@ package org.apache.hadoop.fs;
import java.io.*;
import java.util.Arrays;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -32,7 +30,7 @@ import org.apache.hadoop.util.PureJavaCr
/****************************************************************
* Abstract Checksumed FileSystem.
- * It provide a basice implementation of a Checksumed FileSystem,
+ * It provide a basic implementation of a Checksumed FileSystem,
* which creates a checksum file for each raw file.
* It generates & verifies checksums at the client side.
*
@@ -118,9 +116,6 @@ public abstract class ChecksumFileSystem
* It verifies that data matches checksums.
*******************************************************/
private static class ChecksumFSInputChecker extends FSInputChecker {
- public static final Log LOG
- = LogFactory.getLog(FSInputChecker.class);
-
private ChecksumFileSystem fs;
private FSDataInputStream datas;
private FSDataInputStream sums;
@@ -376,19 +371,6 @@ public abstract class ChecksumFileSystem
public ChecksumFSOutputSummer(ChecksumFileSystem fs,
Path file,
- boolean overwrite,
- short replication,
- long blockSize,
- Configuration conf)
- throws IOException {
- this(fs, file, overwrite,
- conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
- LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT),
- replication, blockSize, null);
- }
-
- public ChecksumFSOutputSummer(ChecksumFileSystem fs,
- Path file,
boolean overwrite,
int bufferSize,
short replication,
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java Fri Mar 1 19:37:03 2013
@@ -17,19 +17,22 @@
*/
package org.apache.hadoop.fs;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.BufferedReader;
-
+import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.NoSuchElementException;
import java.util.StringTokenizer;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.Shell;
+import com.google.common.annotations.VisibleForTesting;
+
/** Filesystem disk space usage statistics.
* Uses the unix 'df' program to get mount points, and java.io.File for
* space utilization. Tested on Linux, FreeBSD, Cygwin. */
@@ -44,6 +47,8 @@ public class DF extends Shell {
private final File dirFile;
private String filesystem;
private String mount;
+
+ private ArrayList<String> output;
enum OSType {
OS_TYPE_UNIX("UNIX"),
@@ -84,6 +89,7 @@ public class DF extends Shell {
super(dfInterval);
this.dirPath = path.getCanonicalPath();
this.dirFile = new File(this.dirPath);
+ this.output = new ArrayList<String>();
}
protected OSType getOSType() {
@@ -127,7 +133,21 @@ public class DF extends Shell {
/** @return the filesystem mount point for the indicated volume */
public String getMount() throws IOException {
+ // Abort early if specified path does not exist
+ if (!dirFile.exists()) {
+ throw new FileNotFoundException("Specified path " + dirFile.getPath()
+ + "does not exist");
+ }
run();
+ // Skip parsing if df was not successful
+ if (getExitCode() != 0) {
+ StringBuffer sb = new StringBuffer("df could not be run successfully: ");
+ for (String line: output) {
+ sb.append(line);
+ }
+ throw new IOException(sb.toString());
+ }
+ parseOutput();
return mount;
}
@@ -164,46 +184,71 @@ public class DF extends Shell {
@Override
protected void parseExecResult(BufferedReader lines) throws IOException {
- lines.readLine(); // skip headings
-
+ output.clear();
String line = lines.readLine();
- if (line == null) {
- throw new IOException( "Expecting a line not the end of stream" );
+ while (line != null) {
+ output.add(line);
+ line = lines.readLine();
+ }
+ }
+
+ @VisibleForTesting
+ protected void parseOutput() throws IOException {
+ if (output.size() < 2) {
+ StringBuffer sb = new StringBuffer("Fewer lines of output than expected");
+ if (output.size() > 0) {
+ sb.append(": " + output.get(0));
+ }
+ throw new IOException(sb.toString());
}
+
+ String line = output.get(1);
StringTokenizer tokens =
new StringTokenizer(line, " \t\n\r\f%");
- this.filesystem = tokens.nextToken();
+ try {
+ this.filesystem = tokens.nextToken();
+ } catch (NoSuchElementException e) {
+ throw new IOException("Unexpected empty line");
+ }
if (!tokens.hasMoreTokens()) { // for long filesystem name
- line = lines.readLine();
- if (line == null) {
- throw new IOException( "Expecting a line not the end of stream" );
+ if (output.size() > 2) {
+ line = output.get(2);
+ } else {
+ throw new IOException("Expecting additional output after line: "
+ + line);
}
tokens = new StringTokenizer(line, " \t\n\r\f%");
}
- switch(getOSType()) {
- case OS_TYPE_AIX:
- Long.parseLong(tokens.nextToken()); // capacity
- Long.parseLong(tokens.nextToken()); // available
- Integer.parseInt(tokens.nextToken()); // pct used
- tokens.nextToken();
- tokens.nextToken();
- this.mount = tokens.nextToken();
- break;
-
- case OS_TYPE_WIN:
- case OS_TYPE_SOLARIS:
- case OS_TYPE_MAC:
- case OS_TYPE_UNIX:
- default:
- Long.parseLong(tokens.nextToken()); // capacity
- Long.parseLong(tokens.nextToken()); // used
- Long.parseLong(tokens.nextToken()); // available
- Integer.parseInt(tokens.nextToken()); // pct used
- this.mount = tokens.nextToken();
- break;
- }
+ try {
+ switch(getOSType()) {
+ case OS_TYPE_AIX:
+ Long.parseLong(tokens.nextToken()); // capacity
+ Long.parseLong(tokens.nextToken()); // available
+ Integer.parseInt(tokens.nextToken()); // pct used
+ tokens.nextToken();
+ tokens.nextToken();
+ this.mount = tokens.nextToken();
+ break;
+
+ case OS_TYPE_WIN:
+ case OS_TYPE_SOLARIS:
+ case OS_TYPE_MAC:
+ case OS_TYPE_UNIX:
+ default:
+ Long.parseLong(tokens.nextToken()); // capacity
+ Long.parseLong(tokens.nextToken()); // used
+ Long.parseLong(tokens.nextToken()); // available
+ Integer.parseInt(tokens.nextToken()); // pct used
+ this.mount = tokens.nextToken();
+ break;
+ }
+ } catch (NoSuchElementException e) {
+ throw new IOException("Could not parse line: " + line);
+ } catch (NumberFormatException e) {
+ throw new IOException("Could not parse line: " + line);
+ }
}
public static void main(String[] args) throws Exception {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Fri Mar 1 19:37:03 2013
@@ -1326,7 +1326,7 @@ public final class FileContext {
*
* 2. Partially qualified URIs (eg scheme but no host)
*
- * fs:///A/B/file Resolved according to the target file sytem. Eg resolving
+ * fs:///A/B/file Resolved according to the target file system. Eg resolving
* a symlink to hdfs:///A results in an exception because
* HDFS URIs must be fully qualified, while a symlink to
* file:///A will not since Hadoop's local file systems
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Fri Mar 1 19:37:03 2013
@@ -1864,7 +1864,7 @@ public abstract class FileSystem extends
*
* Some file systems like LocalFileSystem have an initial workingDir
* that we use as the starting workingDir. For other file systems
- * like HDFS there is no built in notion of an inital workingDir.
+ * like HDFS there is no built in notion of an initial workingDir.
*
* @return if there is built in notion of workingDir then it
* is returned; else a null is returned.
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java Fri Mar 1 19:37:03 2013
@@ -43,7 +43,7 @@ public interface PositionedReadable {
throws IOException;
/**
- * Read number of bytes equalt to the length of the buffer, from a given
+ * Read number of bytes equal to the length of the buffer, from a given
* position within a file. This does not
* change the current offset of a file, and is thread-safe.
*/
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java Fri Mar 1 19:37:03 2013
@@ -79,19 +79,17 @@ public abstract class TrashPolicy extend
/**
* Get an instance of the configured TrashPolicy based on the value
- * of the configuration paramater fs.trash.classname.
+ * of the configuration parameter fs.trash.classname.
*
* @param conf the configuration to be used
* @param fs the file system to be used
* @param home the home directory
* @return an instance of TrashPolicy
*/
- public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home)
- throws IOException {
- Class<? extends TrashPolicy> trashClass = conf.getClass("fs.trash.classname",
- TrashPolicyDefault.class,
- TrashPolicy.class);
- TrashPolicy trash = (TrashPolicy) ReflectionUtils.newInstance(trashClass, conf);
+ public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) {
+ Class<? extends TrashPolicy> trashClass = conf.getClass(
+ "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
+ TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf);
trash.initialize(conf, fs, home); // initialize TrashPolicy
return trash;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java Fri Mar 1 19:37:03 2013
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.
/**
* A byte sequence that is usable as a key or value.
- * It is resizable and distinguishes between the size of the seqeunce and
+ * It is resizable and distinguishes between the size of the sequence and
* the current capacity. The hash function is the front of the md5 of the
* buffer. The sort order is the same as memcmp.
*/
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java Fri Mar 1 19:37:03 2013
@@ -128,7 +128,7 @@ public class Text extends BinaryComparab
/**
* Returns the Unicode Scalar Value (32-bit integer value)
* for the character at <code>position</code>. Note that this
- * method avoids using the converter or doing String instatiation
+ * method avoids using the converter or doing String instantiation
* @return the Unicode scalar value at position or -1
* if the position is invalid or points to a
* trailing byte
@@ -527,7 +527,7 @@ public class Text extends BinaryComparab
int length = 0;
int state = LEAD_BYTE;
while (count < start+len) {
- int aByte = ((int) utf8[count] & 0xFF);
+ int aByte = utf8[count] & 0xFF;
switch (state) {
case LEAD_BYTE:
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Fri Mar 1 19:37:03 2013
@@ -313,6 +313,14 @@ public abstract class Server {
return (addr == null) ? null : addr.getHostAddress();
}
+ /** Returns the RPC remote user when invoked inside an RPC. Note this
+ * may be different than the current user if called within another doAs
+ * @return connection's UGI or null if not an RPC
+ */
+ public static UserGroupInformation getRemoteUser() {
+ Call call = CurCall.get();
+ return (call != null) ? call.connection.user : null;
+ }
/** Return true if the invocation was through an RPC.
*/
@@ -1553,9 +1561,6 @@ public abstract class Server {
UserGroupInformation protocolUser = ProtoUtil.getUgi(connectionContext);
if (saslServer == null) {
user = protocolUser;
- if (user != null) {
- user.setAuthenticationMethod(AuthMethod.SIMPLE);
- }
} else {
// user is authenticated
user.setAuthenticationMethod(authMethod);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java Fri Mar 1 19:37:03 2013
@@ -192,7 +192,7 @@ public class Buffer implements Comparabl
int hash = 1;
byte[] b = this.get();
for (int i = 0; i < count; i++)
- hash = (31 * hash) + (int)b[i];
+ hash = (31 * hash) + b[i];
return hash;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java Fri Mar 1 19:37:03 2013
@@ -26,7 +26,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
/**
- * Interface that alll the serializers have to implement.
+ * Interface that all the serializers have to implement.
*
* @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.
*/
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java Fri Mar 1 19:37:03 2013
@@ -30,6 +30,7 @@ import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.ServiceLoader;
import java.util.Set;
@@ -219,7 +220,7 @@ public class SecurityUtil {
if (fqdn == null || fqdn.isEmpty() || fqdn.equals("0.0.0.0")) {
fqdn = getLocalHostName();
}
- return components[0] + "/" + fqdn.toLowerCase() + "@" + components[2];
+ return components[0] + "/" + fqdn.toLowerCase(Locale.US) + "@" + components[2];
}
static String getLocalHostName() throws UnknownHostException {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java Fri Mar 1 19:37:03 2013
@@ -18,6 +18,11 @@
package org.apache.hadoop.util;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.Enumeration;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -155,5 +160,7 @@ public class VersionInfo {
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());
System.out.println("From source with checksum " + getSrcChecksum());
+ System.out.println("This command was run using " +
+ ClassUtil.findContainingJar(VersionInfo.class));
}
}
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1448457-1451693
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java Fri Mar 1 19:37:03 2013
@@ -17,15 +17,22 @@
*/
package org.apache.hadoop.fs;
-import junit.framework.TestCase;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.StringReader;
import java.util.EnumSet;
+import java.util.Random;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
+import org.junit.Test;
-public class TestDFVariations extends TestCase {
+public class TestDFVariations {
public static class XXDF extends DF {
private final String osName;
@@ -50,6 +57,7 @@ public class TestDFVariations extends Te
}
}
+ @Test(timeout=5000)
public void testOSParsing() throws Exception {
for (DF.OSType ost : EnumSet.allOf(DF.OSType.class)) {
XXDF df = new XXDF(ost.getId());
@@ -58,6 +66,74 @@ public class TestDFVariations extends Te
df.getMount());
}
}
-
+
+ @Test(timeout=5000)
+ public void testDFInvalidPath() throws Exception {
+ // Generate a path that doesn't exist
+ Random random = new Random(0xDEADBEEFl);
+ File file = null;
+ byte[] bytes = new byte[64];
+ while (file == null) {
+ random.nextBytes(bytes);
+ final String invalid = new String("/" + bytes);
+ final File invalidFile = new File(invalid);
+ if (!invalidFile.exists()) {
+ file = invalidFile;
+ }
+ }
+ DF df = new DF(file, 0l);
+ try {
+ df.getMount();
+ } catch (FileNotFoundException e) {
+ // expected, since path does not exist
+ GenericTestUtils.assertExceptionContains(file.getName(), e);
+ }
+ }
+
+ @Test(timeout=5000)
+ public void testDFMalformedOutput() throws Exception {
+ DF df = new DF(new File("/"), 0l);
+ BufferedReader reader = new BufferedReader(new StringReader(
+ "Filesystem 1K-blocks Used Available Use% Mounted on\n" +
+ "/dev/sda5 19222656 10597036 7649060 59% /"));
+ df.parseExecResult(reader);
+ df.parseOutput();
+
+ reader = new BufferedReader(new StringReader(
+ "Filesystem 1K-blocks Used Available Use% Mounted on"));
+ df.parseExecResult(reader);
+ try {
+ df.parseOutput();
+ fail("Expected exception with missing line!");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Fewer lines of output than expected", e);
+ System.out.println(e.toString());
+ }
+
+ reader = new BufferedReader(new StringReader(
+ "Filesystem 1K-blocks Used Available Use% Mounted on\n" +
+ " "));
+ df.parseExecResult(reader);
+ try {
+ df.parseOutput();
+ fail("Expected exception with empty line!");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("Unexpected empty line", e);
+ System.out.println(e.toString());
+ }
+
+ reader = new BufferedReader(new StringReader(
+ "Filesystem 1K-blocks Used Available Use% Mounted on\n" +
+ " 19222656 10597036 7649060 59% /"));
+ df.parseExecResult(reader);
+ try {
+ df.parseOutput();
+ fail("Expected exception with missing field!");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("Could not parse line: ", e);
+ System.out.println(e.toString());
+ }
+ }
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java?rev=1451695&r1=1451694&r2=1451695&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java Fri Mar 1 19:37:03 2013
@@ -127,6 +127,7 @@ public class TestDoAsEffectiveUser {
public static final long versionID = 1L;
String aMethod() throws IOException;
+ String getServerRemoteUser() throws IOException;
}
public class TestImpl implements TestProtocol {
@@ -137,6 +138,11 @@ public class TestDoAsEffectiveUser {
}
@Override
+ public String getServerRemoteUser() throws IOException {
+ return Server.getRemoteUser().toString();
+ }
+
+ @Override
public long getProtocolVersion(String protocol, long clientVersion)
throws IOException {
return TestProtocol.versionID;
@@ -149,7 +155,23 @@ public class TestDoAsEffectiveUser {
}
}
- @Test
+ private void checkRemoteUgi(final Server server,
+ final UserGroupInformation ugi, final Configuration conf)
+ throws Exception {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException {
+ proxy = RPC.getProxy(
+ TestProtocol.class, TestProtocol.versionID,
+ NetUtils.getConnectAddress(server), conf);
+ Assert.assertEquals(ugi.toString(), proxy.aMethod());
+ Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser());
+ return null;
+ }
+ });
+ }
+
+ @Test(timeout=4000)
public void testRealUserSetup() throws IOException {
final Configuration conf = new Configuration();
conf.setStrings(ProxyUsers
@@ -163,24 +185,13 @@ public class TestDoAsEffectiveUser {
try {
server.start();
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
+ checkRemoteUgi(server, realUserUgi, conf);
+
UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- String retVal = proxyUserUgi
- .doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
- }
- });
-
- Assert.assertEquals(PROXY_USER_NAME + " (auth:SIMPLE) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal);
+ checkRemoteUgi(server, proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
@@ -192,7 +203,7 @@ public class TestDoAsEffectiveUser {
}
}
- @Test
+ @Test(timeout=4000)
public void testRealUserAuthorizationSuccess() throws IOException {
final Configuration conf = new Configuration();
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
@@ -206,25 +217,13 @@ public class TestDoAsEffectiveUser {
try {
server.start();
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
+ checkRemoteUgi(server, realUserUgi, conf);
UserGroupInformation proxyUserUgi = UserGroupInformation
.createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- String retVal = proxyUserUgi
- .doAs(new PrivilegedExceptionAction<String>() {
- @Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
- }
- });
-
- Assert.assertEquals(PROXY_USER_NAME + " (auth:SIMPLE) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal);
+ checkRemoteUgi(server, proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();