You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2011/05/24 00:20:49 UTC
svn commit: r1126786 - in /hadoop/common/branches/yahoo-merge: ./
src/contrib/ec2/ src/docs/ src/java/ src/java/org/apache/hadoop/fs/
src/java/org/apache/hadoop/fs/shell/ src/test/core/
src/test/core/org/apache/hadoop/cli/
Author: szetszwo
Date: Mon May 23 22:20:48 2011
New Revision: 1126786
URL: http://svn.apache.org/viewvc?rev=1126786&view=rev
Log:
svn merge -c 1099633 from trunk for HADOOP-7250.
Added:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/SetReplication.java
- copied unchanged from r1099633, hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/SetReplication.java
Modified:
hadoop/common/branches/yahoo-merge/ (props changed)
hadoop/common/branches/yahoo-merge/CHANGES.txt (contents, props changed)
hadoop/common/branches/yahoo-merge/src/contrib/ec2/ (props changed)
hadoop/common/branches/yahoo-merge/src/docs/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
hadoop/common/branches/yahoo-merge/src/test/core/ (props changed)
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml
Propchange: hadoop/common/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 22:20:48 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1100026
+/hadoop/common/trunk:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026
/hadoop/core/branches/branch-0.19/core:713112
Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1126786&r1=1126785&r2=1126786&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Mon May 23 22:20:48 2011
@@ -41,6 +41,9 @@ Trunk (unreleased changes)
HADOOP-7236. Refactor the mkdir command to conform to new FsCommand class.
(Daryn Sharp via szetszwo)
+ HADOOP-7250. Refactor the setrep command to conform to new FsCommand class.
+ (Daryn Sharp via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
Propchange: hadoop/common/branches/yahoo-merge/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 22:20:48 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1100026
+/hadoop/common/trunk/CHANGES.txt:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/yahoo-merge/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 22:20:48 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/contrib/ec2:1043117,1078148,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1100026
+/hadoop/common/trunk/src/contrib/ec2:1043117,1078148,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026
/hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
/hadoop/core/trunk/src/contrib/ec2:776175-784663
Propchange: hadoop/common/branches/yahoo-merge/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 22:20:48 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/src/docs:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1100026
+/hadoop/common/trunk/src/docs:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 22:20:48 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1100026
+/hadoop/common/trunk/src/java:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java?rev=1126786&r1=1126785&r2=1126786&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java Mon May 23 22:20:48 2011
@@ -68,7 +68,6 @@ public class FsShell extends Configured
static {
modifFmt.setTimeZone(TimeZone.getTimeZone("UTC"));
}
- static final String SETREP_SHORT_USAGE="-setrep [-R] [-w] <rep> <path/file>";
static final String GET_SHORT_USAGE = "-get [-ignoreCrc] [-crc] <src> <localdst>";
static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace(
"-get", "-copyToLocal");
@@ -463,150 +462,6 @@ public class FsShell extends Configured
}
}.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
}
-
- /**
- * Parse the incoming command string
- * @param cmd
- * @param pos ignore anything before this pos in cmd
- * @throws IOException
- */
- private void setReplication(String[] cmd, int pos) throws IOException {
- CommandFormat c = new CommandFormat("setrep", 2, 2, "R", "w");
- String dst = null;
- short rep = 0;
-
- try {
- List<String> parameters = c.parse(cmd, pos);
- rep = Short.parseShort(parameters.get(0));
- dst = parameters.get(1);
- } catch (NumberFormatException nfe) {
- System.err.println("Illegal replication, a positive integer expected");
- throw nfe;
- }
- catch(IllegalArgumentException iae) {
- System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
- throw iae;
- }
-
- if (rep < 1) {
- System.err.println("Cannot set replication to: " + rep);
- throw new IllegalArgumentException("replication must be >= 1");
- }
-
- List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
- setReplication(rep, dst, c.getOpt("R"), waitList);
-
- if (waitList != null) {
- waitForReplication(waitList, rep);
- }
- }
-
- /**
- * Wait for all files in waitList to have replication number equal to rep.
- * @param waitList The files are waited for.
- * @param rep The new replication number.
- * @throws IOException IOException
- */
- void waitForReplication(List<Path> waitList, int rep) throws IOException {
- for(Path f : waitList) {
- System.out.print("Waiting for " + f + " ...");
- System.out.flush();
-
- boolean printWarning = false;
- FileSystem pFS = f.getFileSystem(getConf());
- FileStatus status = pFS.getFileStatus(f);
- long len = status.getLen();
-
- for(boolean done = false; !done; ) {
- BlockLocation[] locations = pFS.getFileBlockLocations(status, 0, len);
- int i = 0;
- for(; i < locations.length &&
- locations[i].getHosts().length == rep; i++)
- if (!printWarning && locations[i].getHosts().length > rep) {
- System.out.println("\nWARNING: the waiting time may be long for "
- + "DECREASING the number of replication.");
- printWarning = true;
- }
- done = i == locations.length;
-
- if (!done) {
- System.out.print(".");
- System.out.flush();
- try {Thread.sleep(10000);} catch (InterruptedException e) {}
- }
- }
-
- System.out.println(" done");
- }
- }
-
- /**
- * Set the replication for files that match file pattern <i>srcf</i>
- * if it's a directory and recursive is true,
- * set replication for all the subdirs and those files too.
- * @param newRep new replication factor
- * @param srcf a file pattern specifying source files
- * @param recursive if need to set replication factor for files in subdirs
- * @throws IOException
- * @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
- */
- void setReplication(short newRep, String srcf, boolean recursive,
- List<Path> waitingList)
- throws IOException {
- Path srcPath = new Path(srcf);
- FileSystem srcFs = srcPath.getFileSystem(getConf());
- Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath),
- srcPath);
- for(int i=0; i<srcs.length; i++) {
- setReplication(newRep, srcFs, srcs[i], recursive, waitingList);
- }
- }
-
- private void setReplication(short newRep, FileSystem srcFs,
- Path src, boolean recursive,
- List<Path> waitingList)
- throws IOException {
- if (srcFs.getFileStatus(src).isFile()) {
- setFileReplication(src, srcFs, newRep, waitingList);
- return;
- }
- FileStatus items[];
- try {
- items = srcFs.listStatus(src);
- } catch (FileNotFoundException fnfe) {
- throw new IOException("Could not get listing for " + src);
- }
-
- for (int i = 0; i < items.length; i++) {
- if (items[i].isFile()) {
- setFileReplication(items[i].getPath(), srcFs, newRep, waitingList);
- } else if (items[i].isSymlink()) {
- throw new AssertionError("Symlinks unsupported");
- } else if (recursive) {
- setReplication(newRep, srcFs, items[i].getPath(), recursive,
- waitingList);
- }
- }
- }
-
- /**
- * Actually set the replication for this file
- * If it fails either throw IOException or print an error msg
- * @param file a file/directory
- * @param newRep new replication factor
- * @throws IOException on error
- */
- private void setFileReplication(Path file, FileSystem srcFs, short newRep, List<Path> waitList)
- throws IOException {
- if (srcFs.setReplication(file, newRep)) {
- if (waitList != null) {
- waitList.add(file);
- }
- System.out.println("Replication " + newRep + " set: " + file);
- } else {
- System.err.println("Could not set replication for: " + file);
- }
- }
/**
* Show the size of a partition in the filesystem that contains
@@ -1237,7 +1092,7 @@ public class FsShell extends Configured
GET_SHORT_USAGE + "\n\t" +
"[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" +
"[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
- "[-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
+ "[-report]\n\t" +
"[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" +
"[-text <path>]\n\t" +
"[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" +
@@ -1327,11 +1182,6 @@ public class FsShell extends Configured
String moveToLocal = "-moveToLocal <src> <localdst>: Not implemented yet \n";
- String setrep = SETREP_SHORT_USAGE
- + ": Set the replication level of a file. \n"
- + "\t\tThe -R flag requests a recursive change of replication level \n"
- + "\t\tfor an entire tree.\n";
-
String touchz = "-touchz <path>: Creates a file of zero length\n"
+ "\t\t at <path> with current time as the timestamp of that <path>.\n"
+ "\t\t An error is returned if the file exists with non-zero length\n";
@@ -1423,8 +1273,6 @@ public class FsShell extends Configured
System.out.println(cat);
} else if ("get".equals(cmd)) {
System.out.println(get);
- } else if ("setrep".equals(cmd)) {
- System.out.println(setrep);
} else if ("touchz".equals(cmd)) {
System.out.println(touchz);
} else if ("test".equals(cmd)) {
@@ -1465,7 +1313,6 @@ public class FsShell extends Configured
System.out.println(cat);
System.out.println(copyToLocal);
System.out.println(moveToLocal);
- System.out.println(setrep);
System.out.println(touchz);
System.out.println(test);
System.out.println(text);
@@ -1613,8 +1460,6 @@ public class FsShell extends Configured
} else if ("-cat".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " <src>]");
- } else if ("-setrep".equals(cmd)) {
- System.err.println("Usage: java FsShell [" + SETREP_SHORT_USAGE + "]");
} else if ("-test".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-test -[ezd] <path>]");
@@ -1640,7 +1485,6 @@ public class FsShell extends Configured
System.err.println(" [-text <src>]");
System.err.println(" [" + COPYTOLOCAL_SHORT_USAGE + "]");
System.err.println(" [-moveToLocal [-crc] <src> <localdst>]");
- System.err.println(" [" + SETREP_SHORT_USAGE + "]");
System.err.println(" [-touchz <path>]");
System.err.println(" [-test -[ezd] <path>]");
System.err.println(" [-stat [format] <path>]");
@@ -1754,8 +1598,6 @@ public class FsShell extends Configured
exitCode = doall(cmd, argv, i);
} else if ("-moveToLocal".equals(cmd)) {
moveToLocal(argv[i++], new Path(argv[i++]));
- } else if ("-setrep".equals(cmd)) {
- setReplication(argv, i);
} else if ("-chmod".equals(cmd) ||
"-chown".equals(cmd) ||
"-chgrp".equals(cmd)) {
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java?rev=1126786&r1=1126785&r2=1126786&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java Mon May 23 22:20:48 2011
@@ -45,6 +45,7 @@ abstract public class FsCommand extends
factory.registerCommands(Count.class);
factory.registerCommands(Ls.class);
factory.registerCommands(Mkdir.class);
+ factory.registerCommands(SetReplication.class);
factory.registerCommands(Tail.class);
}
Propchange: hadoop/common/branches/yahoo-merge/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 23 22:20:48 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/test/core:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1100026
+/hadoop/common/trunk/src/test/core:1043117,1078148,1080396,1081598,1082329,1082787-1082788,1084769,1085043,1085122,1086309,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml?rev=1126786&r1=1126785&r2=1126786&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml (original)
+++ hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml Mon May 23 22:20:48 2011
@@ -518,7 +518,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-setrep \[-R\] \[-w\] <rep> <path/file>:( )*Set the replication level of a file.( )*</expected-output>
+ <expected-output>^-setrep \[-R\] \[-w\] <rep> <path/file> \.\.\.:( |\t)*Set the replication level of a file.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>