You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2014/02/21 21:08:09 UTC
svn commit: r1570694 - in
/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common: ./
src/main/java/ src/main/java/org/apache/hadoop/fs/
src/main/java/org/apache/hadoop/fs/permission/
src/main/java/org/apache/hadoop/fs/shell/ src/main/jav...
Author: szetszwo
Date: Fri Feb 21 20:08:08 2014
New Revision: 1570694
URL: http://svn.apache.org/r1570694
Log:
Merge r1569890 through r1570692 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestProgress.java
- copied unchanged from r1570692, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestProgress.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt Fri Feb 21 20:08:08 2014
@@ -326,6 +326,11 @@ Trunk (Unreleased)
HADOOP-10344. Fix TestAclCommands after merging HADOOP-10338 patch.
(cnauroth)
+ HADOOP-10352. Recursive setfacl erroneously attempts to apply default ACL to
+ files. (cnauroth)
+
+ HADOOP-10354. TestWebHDFS fails after merge of HDFS-4685 to trunk. (cnauroth)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -396,6 +401,8 @@ Release 2.4.0 - UNRELEASED
HADOOP-10328. loadGenerator exit code is not reliable.
(Haohui Mai via cnauroth)
+ HADOOP-10355. Fix TestLoadGenerator#testLoadGenerator. (Haohui Mai via jing9)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1570392-1570692
Propchange: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1570392-1570692
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java Fri Feb 21 20:08:08 2014
@@ -569,6 +569,11 @@ public class RawLocalFileSystem extends
//expected format
//-rw------- 1 username groupname ...
String permission = t.nextToken();
+ if (permission.length() > FsPermission.MAX_PERMISSION_LENGTH) {
+ //files with ACLs might have a '+'
+ permission = permission.substring(0,
+ FsPermission.MAX_PERMISSION_LENGTH);
+ }
setPermission(FsPermission.valueOf(permission));
t.nextToken();
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java Fri Feb 21 20:08:08 2014
@@ -48,6 +48,9 @@ public class FsPermission implements Wri
WritableFactories.setFactory(ImmutableFsPermission.class, FACTORY);
}
+ /** Maximum acceptable length of a permission string to parse */
+ public static final int MAX_PERMISSION_LENGTH = 10;
+
/** Create an immutable {@link FsPermission} object. */
public static FsPermission createImmutable(short permission) {
return new ImmutableFsPermission(permission);
@@ -319,9 +322,10 @@ public class FsPermission implements Wri
if (unixSymbolicPermission == null) {
return null;
}
- else if (unixSymbolicPermission.length() != 10) {
- throw new IllegalArgumentException("length != 10(unixSymbolicPermission="
- + unixSymbolicPermission + ")");
+ else if (unixSymbolicPermission.length() != MAX_PERMISSION_LENGTH) {
+ throw new IllegalArgumentException(String.format(
+ "length != %d(unixSymbolicPermission=%s)", MAX_PERMISSION_LENGTH,
+ unixSymbolicPermission));
}
int n = 0;
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java Fri Feb 21 20:08:08 2014
@@ -22,6 +22,8 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import com.google.common.collect.Lists;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -231,6 +233,7 @@ class AclCommands extends FsCommand {
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "b", "k", "R",
"m", "x", "-set");
List<AclEntry> aclEntries = null;
+ List<AclEntry> accessAclEntries = null;
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
@@ -263,6 +266,19 @@ class AclCommands extends FsCommand {
if (args.size() > 1) {
throw new HadoopIllegalArgumentException("Too many arguments");
}
+
+ // In recursive mode, save a separate list of just the access ACL entries.
+ // Only directories may have a default ACL. When a recursive operation
+ // encounters a file under the specified path, it must pass only the
+ // access ACL entries.
+ if (isRecursive() && (oneModifyOption || setOption)) {
+ accessAclEntries = Lists.newArrayList();
+ for (AclEntry entry: aclEntries) {
+ if (entry.getScope() == AclEntryScope.ACCESS) {
+ accessAclEntries.add(entry);
+ }
+ }
+ }
}
@Override
@@ -272,11 +288,37 @@ class AclCommands extends FsCommand {
} else if (cf.getOpt("k")) {
item.fs.removeDefaultAcl(item.path);
} else if (cf.getOpt("m")) {
- item.fs.modifyAclEntries(item.path, aclEntries);
+ List<AclEntry> entries = getAclEntries(item);
+ if (!entries.isEmpty()) {
+ item.fs.modifyAclEntries(item.path, entries);
+ }
} else if (cf.getOpt("x")) {
- item.fs.removeAclEntries(item.path, aclEntries);
+ List<AclEntry> entries = getAclEntries(item);
+ if (!entries.isEmpty()) {
+ item.fs.removeAclEntries(item.path, entries);
+ }
} else if (cf.getOpt("-set")) {
- item.fs.setAcl(item.path, aclEntries);
+ List<AclEntry> entries = getAclEntries(item);
+ if (!entries.isEmpty()) {
+ item.fs.setAcl(item.path, entries);
+ }
+ }
+ }
+
+ /**
+ * Returns the ACL entries to use in the API call for the given path. For a
+ * recursive operation, returns all specified ACL entries if the item is a
+ * directory or just the access ACL entries if the item is a file. For a
+ * non-recursive operation, returns all specified ACL entries.
+ *
+ * @param item PathData path to check
+ * @return List<AclEntry> ACL entries to use in the API call
+ */
+ private List<AclEntry> getAclEntries(PathData item) {
+ if (isRecursive()) {
+ return item.stat.isDirectory() ? aclEntries : accessAclEntries;
+ } else {
+ return aclEntries;
}
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java Fri Feb 21 20:08:08 2014
@@ -160,6 +160,31 @@ public class Progress {
/** Called during execution on a leaf node to set its progress. */
public synchronized void set(float progress) {
+ if (Float.isNaN(progress)) {
+ progress = 0;
+ LOG.warn("Illegal progress value found, progress is Float.NaN. " +
+ "Progress will be changed to 0");
+ }
+ else if (progress == Float.NEGATIVE_INFINITY) {
+ progress = 0;
+ LOG.warn("Illegal progress value found, progress is " +
+ "Float.NEGATIVE_INFINITY. Progress will be changed to 0");
+ }
+ else if (progress < 0) {
+ progress = 0;
+ LOG.warn("Illegal progress value found, progress is less than 0." +
+ " Progress will be changed to 0");
+ }
+ else if (progress > 1) {
+ progress = 1;
+ LOG.warn("Illegal progress value found, progress is larger than 1." +
+ " Progress will be changed to 1");
+ }
+ else if (progress == Float.POSITIVE_INFINITY) {
+ progress = 1;
+ LOG.warn("Illegal progress value found, progress is " +
+ "Float.POSITIVE_INFINITY. Progress will be changed to 1");
+ }
this.progress = progress;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java Fri Feb 21 20:08:08 2014
@@ -45,6 +45,8 @@ import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import com.google.common.base.Preconditions;
+
/** The load generator is a tool for testing NameNode behavior under
* different client loads.
* It allows the user to generate different mixes of read, write,
@@ -488,7 +490,35 @@ public class LoadGenerator extends Confi
return initFileDirTables();
}
-
+
+ private static void parseScriptLine(String line, ArrayList<Long> duration,
+ ArrayList<Double> readProb, ArrayList<Double> writeProb) {
+ String[] a = line.split("\\s");
+
+ if (a.length != 3) {
+ throw new IllegalArgumentException("Incorrect number of parameters: "
+ + line);
+ }
+
+ try {
+ long d = Long.parseLong(a[0]);
+ double r = Double.parseDouble(a[1]);
+ double w = Double.parseDouble(a[2]);
+
+ Preconditions.checkArgument(d >= 0, "Invalid duration: " + d);
+ Preconditions.checkArgument(0 <= r && r <= 1.0,
+ "The read probability must be [0, 1]: " + r);
+ Preconditions.checkArgument(0 <= w && w <= 1.0,
+ "The read probability must be [0, 1]: " + w);
+
+ readProb.add(r);
+ duration.add(d);
+ writeProb.add(w);
+ } catch (NumberFormatException nfe) {
+ throw new IllegalArgumentException("Cannot parse: " + line);
+ }
+ }
+
/**
* Read a script file of the form: lines of text with duration in seconds,
* read probability and write probability, separated by white space.
@@ -508,47 +538,19 @@ public class LoadGenerator extends Confi
String line;
// Read script, parse values, build array of duration, read and write probs
- while ((line = br.readLine()) != null) {
- lineNum++;
- if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
- continue;
-
- String[] a = line.split("\\s");
- if (a.length != 3) {
- System.err.println("Line " + lineNum
- + ": Incorrect number of parameters: " + line);
- }
-
- try {
- long d = Long.parseLong(a[0]);
- if (d < 0) {
- System.err.println("Line " + lineNum + ": Invalid duration: " + d);
- return -1;
- }
-
- double r = Double.parseDouble(a[1]);
- if (r < 0.0 || r > 1.0) {
- System.err.println("Line " + lineNum
- + ": The read probability must be [0, 1]: " + r);
- return -1;
- }
-
- double w = Double.parseDouble(a[2]);
- if (w < 0.0 || w > 1.0) {
- System.err.println("Line " + lineNum
- + ": The read probability must be [0, 1]: " + r);
- return -1;
- }
+ try {
+ while ((line = br.readLine()) != null) {
+ lineNum++;
+ if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
+ continue;
- readProb.add(r);
- duration.add(d);
- writeProb.add(w);
- } catch (NumberFormatException nfe) {
- System.err.println(lineNum + ": Can't parse: " + line);
- return -1;
- } finally {
- IOUtils.cleanup(LOG, br);
+ parseScriptLine(line, duration, readProb, writeProb);
}
+ } catch (IllegalArgumentException e) {
+ System.err.println("Line: " + lineNum + ", " + e.getMessage());
+ return -1;
+ } finally {
+ IOUtils.cleanup(LOG, br);
}
// Copy vectors to arrays of values, to avoid autoboxing overhead later