You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2011/04/22 22:43:56 UTC
svn commit: r1096030 - in /hadoop/common/branches/yahoo-merge: ./
src/contrib/ec2/ src/docs/ src/java/ src/test/core/
src/test/core/org/apache/hadoop/fs/loadGenerator/
Author: suresh
Date: Fri Apr 22 20:43:55 2011
New Revision: 1096030
URL: http://svn.apache.org/viewvc?rev=1096030&view=rev
Log:
Merging change r1043117 from trunk
Modified:
hadoop/common/branches/yahoo-merge/ (props changed)
hadoop/common/branches/yahoo-merge/CHANGES.txt (contents, props changed)
hadoop/common/branches/yahoo-merge/src/contrib/ec2/ (props changed)
hadoop/common/branches/yahoo-merge/src/docs/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/ (props changed)
hadoop/common/branches/yahoo-merge/src/test/core/ (props changed)
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
Propchange: hadoop/common/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 22 20:43:55 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:1080396
+/hadoop/common/trunk:1043117,1080396
/hadoop/core/branches/branch-0.19/core:713112
Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1096030&r1=1096029&r2=1096030&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Fri Apr 22 20:43:55 2011
@@ -15,6 +15,9 @@ Trunk (unreleased changes)
HADOOP-7133. Batch the calls in DataStorage to FileUtil.createHardLink().
(Matt Foley via jghoman)
+ HADOOP-7054 Change NN LoadGenerator to use FileContext APIs
+ (Sanjay Radia)
+
OPTIMIZATIONS
BUG FIXES
Propchange: hadoop/common/branches/yahoo-merge/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 22 20:43:55 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:1080396,1091618,1092832
+/hadoop/common/trunk/CHANGES.txt:1043117,1080396,1091618,1092832
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/yahoo-merge/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 22 20:43:55 2011
@@ -1,2 +1,3 @@
+/hadoop/common/trunk/src/contrib/ec2:1043117
/hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
/hadoop/core/trunk/src/contrib/ec2:776175-784663
Propchange: hadoop/common/branches/yahoo-merge/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 22 20:43:55 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/src/docs:1080396
+/hadoop/common/trunk/src/docs:1043117,1080396
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 22 20:43:55 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:1080396
+/hadoop/common/trunk/src/java:1043117,1080396
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Propchange: hadoop/common/branches/yahoo-merge/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Apr 22 20:43:55 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/test/core:1080396
+/hadoop/common/trunk/src/test/core:1043117,1080396
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java?rev=1096030&r1=1096029&r2=1096030&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java (original)
+++ hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java Fri Apr 22 20:43:55 2011
@@ -22,12 +22,15 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -48,7 +51,7 @@ import org.apache.hadoop.util.ToolRunner
public class DataGenerator extends Configured implements Tool {
private File inDir = StructureGenerator.DEFAULT_STRUCTURE_DIRECTORY;
private Path root = DEFAULT_ROOT;
- private FileSystem fs;
+ private FileContext fc;
final static private long BLOCK_SIZE = 10;
final static private String USAGE = "java DataGenerator " +
"-inDir <inDir> " +
@@ -78,7 +81,7 @@ public class DataGenerator extends Confi
/** Parse the command line arguments and initialize the data */
private int init(String[] args) {
try { // initialize file system handle
- fs = FileSystem.get(getConf());
+ fc = FileContext.getFileContext(getConf());
} catch (IOException ioe) {
System.err.println("Can not initialize the file system: " +
ioe.getLocalizedMessage());
@@ -109,7 +112,7 @@ public class DataGenerator extends Confi
StructureGenerator.DIR_STRUCTURE_FILE_NAME)));
String line;
while ((line=in.readLine()) != null) {
- fs.mkdirs(new Path(root+line));
+ fc.mkdir(new Path(root+line), FileContext.DEFAULT_PERM, true);
}
}
@@ -137,10 +140,9 @@ public class DataGenerator extends Confi
* a length of <code>fileSize</code>. The file is filled with character 'a'.
*/
private void genFile(Path file, long fileSize) throws IOException {
- FSDataOutputStream out = fs.create(file, true,
- getConf().getInt("io.file.buffer.size", 4096),
- (short)getConf().getInt("dfs.replication", 3),
- fs.getDefaultBlockSize());
+ FSDataOutputStream out = fc.create(file, EnumSet.of(CreateFlag.OVERWRITE),
+ CreateOpts.createParent(), CreateOpts.bufferSize(4096),
+ CreateOpts.repFac((short) 3));
for(long i=0; i<fileSize; i++) {
out.writeByte('a');
}
Modified: hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java?rev=1096030&r1=1096029&r2=1096030&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java (original)
+++ hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java Fri Apr 22 20:43:55 2011
@@ -26,16 +26,19 @@ import java.io.InputStream;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -110,7 +113,7 @@ public class LoadGenerator extends Confi
private volatile boolean shouldRun = true;
private Path root = DataGenerator.DEFAULT_ROOT;
- private FileSystem fs;
+ private FileContext fc;
private int maxDelayBetweenOps = 0;
private int numOfThreads = 200;
private long [] durations = {0};
@@ -230,7 +233,7 @@ public class LoadGenerator extends Confi
private void read() throws IOException {
String fileName = files.get(r.nextInt(files.size()));
long startTime = System.currentTimeMillis();
- InputStream in = fs.open(new Path(fileName));
+ InputStream in = fc.open(new Path(fileName));
executionTime[OPEN] += (System.currentTimeMillis()-startTime);
totalNumOfOps[OPEN]++;
while (in.read(buffer) != -1) {}
@@ -252,7 +255,7 @@ public class LoadGenerator extends Confi
while ((fileSize = r.nextGaussian()+2)<=0) {}
genFile(file, (long)(fileSize*BLOCK_SIZE));
long startTime = System.currentTimeMillis();
- fs.delete(file, true);
+ fc.delete(file, true);
executionTime[DELETE] += (System.currentTimeMillis()-startTime);
totalNumOfOps[DELETE]++;
}
@@ -263,7 +266,7 @@ public class LoadGenerator extends Confi
private void list() throws IOException {
String dirName = dirs.get(r.nextInt(dirs.size()));
long startTime = System.currentTimeMillis();
- fs.listStatus(new Path(dirName));
+ fc.listStatus(new Path(dirName));
executionTime[LIST] += (System.currentTimeMillis()-startTime);
totalNumOfOps[LIST]++;
}
@@ -352,7 +355,7 @@ public class LoadGenerator extends Confi
/** Parse the command line arguments and initialize the data */
private int init(String[] args) throws IOException {
try {
- fs = FileSystem.get(getConf());
+ fc = FileContext.getFileContext(getConf());
} catch (IOException ioe) {
System.err.println("Can not initialize the file system: " +
ioe.getLocalizedMessage());
@@ -546,7 +549,7 @@ public class LoadGenerator extends Confi
* whose name starts with "_file_".
*/
private void initFileDirTables(Path path) throws IOException {
- FileStatus[] stats = fs.listStatus(path);
+ FileStatus[] stats = fc.util().listStatus(path);
for (FileStatus stat : stats) {
if (stat.isDirectory()) {
@@ -581,10 +584,9 @@ public class LoadGenerator extends Confi
*/
private void genFile(Path file, long fileSize) throws IOException {
long startTime = System.currentTimeMillis();
- FSDataOutputStream out = fs.create(file, true,
- getConf().getInt("io.file.buffer.size", 4096),
- (short)getConf().getInt("dfs.replication", 3),
- fs.getDefaultBlockSize());
+ FSDataOutputStream out = fc.create(file, EnumSet.of(CreateFlag.OVERWRITE),
+ CreateOpts.createParent(), CreateOpts.bufferSize(4096),
+ CreateOpts.repFac((short) 3));
executionTime[CREATE] += (System.currentTimeMillis()-startTime);
totalNumOfOps[CREATE]++;