You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-user@hadoop.apache.org by "11 Nov." <no...@gmail.com> on 2008/04/08 04:58:45 UTC
Can't get DFS file size
Hi folks,
I'm writing a little test programm to check the writing speed of DFS
file system, but can't get the file size using
"fs.getFileStatus(file).getLen()" or fs.getContentLength(file). Here is my
code:
package org.apache.hadoop.fs;
import java.io.IOException;
import java.io.OutputStream;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
public class TestDFSWrite extends TestCase {
static String ROOT = System.getProperty("test.build.data","fs_test");
static Path DATA_DIR = new Path(ROOT, "fs_data");
static long MEGA = 1024 * 1024;
static int BUFFER_SIZE = 4096;
static Configuration conf = new Configuration();
static FileSystem fs;
static byte[] buffer = new byte[BUFFER_SIZE];
static boolean finished;
public class FileStatusChecker extends Thread{
Path file;
Path resultFile;
int interval;
public FileStatusChecker(Path file, Path resultFile, int interval){
this.file = file;
this.resultFile = resultFile;
this.interval = interval;
}
public void run(){
System.out.println("Here is the checker running!!");
System.out.println(file.toString());
SequenceFile.Writer writer = null;
try {
long lastLen = 1;
long thisLen = 1;
writer = SequenceFile.createWriter(fs, conf, resultFile,
Text.class, Text.class,
CompressionType.NONE);
while(!finished){
lastLen = thisLen;
if(fs.exists(file)){
System.out.println("File exists!");
thisLen = fs.getContentLength(file);
System.out.println("@@"+thisLen);
}
else{
sleep(interval * 10);
continue;
}
long length = thisLen -lastLen;
System.out.println("thisLen is" + thisLen + "lastLen is " +
lastLen);
long cur = System.currentTimeMillis();
cur = cur - (cur % 10);
LongWritable current = new LongWritable(cur);
writer.append(new Text(current.toString()), new Text(new
LongWritable(length).toString()));
sleep(interval * 10);
}
return;
}catch (Exception e) {
e.printStackTrace();
}finally {
try {
writer.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
public static void main(String[] args) throws Exception {
{
try {
fs = FileSystem.get(conf);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
String testFunc = "";
String fileName = "";
String resultFileName = "";
int fileSize = 0;
int interval = 0;
String usage = "Usage: TestDFSWrite -testfunc [read/write] -file foo
-size M -interval MS -result resultFile";
if (args.length == 0) {
System.err.println(usage);
System.exit(-1);
}
for (int i = 0; i < args.length; i++) { // parse command line
if (args[i].equals("-testfunc")) {
testFunc = args[++i];
} else if (args[i].equals("-file")) {
fileName = args[++i];
} else if (args[i].equals("-size")) {
fileSize = Integer.parseInt(args[++i]);
} else if (args[i].equals("-interval")) {
interval = Integer.parseInt(args[++i]);
} else if (args[i].equals("-result")) {
resultFileName = args[++i];
}
}
long total = fileSize * MEGA;
OutputStream out;
Path file, resultFile;
fs.delete(DATA_DIR);
if(testFunc.equals("read")){
System.out.println("This option is not ready.");
return;
}else if(testFunc.equals("write")){
file = new Path(DATA_DIR, fileName);
resultFile = new Path(DATA_DIR, resultFileName);
}else{
System.out.println("Invalid command line option.");
return;
}
FileStatusChecker checker = new TestDFSWrite().new
FileStatusChecker(file, resultFile, interval);
System.out.println(file.toString());
//System.out.println("F L:" +
newLongWritable(fs.getContentLength(file)).toString());
out = fs.create(file);
checker.start();
long written = 0;
try {
finished = false;
while (written < total) {
long remains = total - written;
int length = (remains<=buffer.length) ? (int)remains :
buffer.length;
out.write(buffer, 0, length);
out.flush();
written += length;
System.out.println("One segment done!");
System.out.println("F L:" + new
LongWritable(fs.getFileStatus(file).getLen()).toString());
}
finished =true;
} finally {
out.close();
}
//System.exit(0);
}
}
Neither the inner class FileStatusChecker nor the main function can get the
right size, they both get "0".
What should be the problem?
Re: Can't get DFS file size
Posted by "11 Nov." <no...@gmail.com>.
I tried to play with the little test by attaching eclipse on when it
started, what surprised me is that the size could be got in eclipse, and the
result file is witten as expected. Can anybody explain this?
2008/4/8, 11 Nov. <no...@gmail.com>:
>
> Hi folks,
> I'm writing a little test programm to check the writing speed of DFS
> file system, but can't get the file size using
> "fs.getFileStatus(file).getLen()" or fs.getContentLength(file). Here is my
> code:
>
> package org.apache.hadoop.fs;
> import java.io.IOException;
> import java.io.OutputStream;
>
> import junit.framework.TestCase;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.fs.FileSystem;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.SequenceFile;
> import org.apache.hadoop.io.SequenceFile.CompressionType;
> import org.apache.hadoop.io.Text;
>
> public class TestDFSWrite extends TestCase {
> static String ROOT = System.getProperty("test.build.data","fs_test");
> static Path DATA_DIR = new Path(ROOT, "fs_data");
> static long MEGA = 1024 * 1024;
> static int BUFFER_SIZE = 4096;
> static Configuration conf = new Configuration();
> static FileSystem fs;
>
> static byte[] buffer = new byte[BUFFER_SIZE];
> static boolean finished;
>
> public class FileStatusChecker extends Thread{
> Path file;
> Path resultFile;
> int interval;
>
> public FileStatusChecker(Path file, Path resultFile, int interval){
> this.file = file;
> this.resultFile = resultFile;
> this.interval = interval;
> }
>
> public void run(){
> System.out.println("Here is the checker running!!");
> System.out.println(file.toString());
> SequenceFile.Writer writer = null;
> try {
> long lastLen = 1;
> long thisLen = 1;
> writer = SequenceFile.createWriter(fs, conf, resultFile,
> Text.class, Text.class,
> CompressionType.NONE);
>
> while(!finished){
> lastLen = thisLen;
> if(fs.exists(file)){
> System.out.println("File exists!");
> thisLen = fs.getContentLength(file);
> System.out.println("@@"+thisLen);
> }
> else{
> sleep(interval * 10);
> continue;
> }
> long length = thisLen -lastLen;
> System.out.println("thisLen is" + thisLen + "lastLen is " +
> lastLen);
> long cur = System.currentTimeMillis();
> cur = cur - (cur % 10);
> LongWritable current = new LongWritable(cur);
> writer.append(new Text(current.toString()), new Text(new
> LongWritable(length).toString()));
> sleep(interval * 10);
>
> }
> return;
>
> }catch (Exception e) {
> e.printStackTrace();
> }finally {
> try {
> writer.close();
> } catch (Exception e) {
> e.printStackTrace();
> }
> }
> }
> }
>
> public static void main(String[] args) throws Exception {
> {
> try {
> fs = FileSystem.get(conf);
> } catch (IOException e) {
> throw new RuntimeException(e);
> }
> }
> String testFunc = "";
> String fileName = "";
> String resultFileName = "";
> int fileSize = 0;
> int interval = 0;
>
> String usage = "Usage: TestDFSWrite -testfunc [read/write] -file foo
> -size M -interval MS -result resultFile";
> if (args.length == 0) {
> System.err.println(usage);
> System.exit(-1);
> }
>
> for (int i = 0; i < args.length; i++) { // parse command line
> if (args[i].equals("-testfunc")) {
> testFunc = args[++i];
> } else if (args[i].equals("-file")) {
> fileName = args[++i];
> } else if (args[i].equals("-size")) {
> fileSize = Integer.parseInt(args[++i]);
> } else if (args[i].equals("-interval")) {
> interval = Integer.parseInt(args[++i]);
> } else if (args[i].equals("-result")) {
> resultFileName = args[++i];
> }
> }
>
> long total = fileSize * MEGA;
> OutputStream out;
> Path file, resultFile;
> fs.delete(DATA_DIR);
>
> if(testFunc.equals("read")){
> System.out.println("This option is not ready.");
> return;
> }else if(testFunc.equals("write")){
> file = new Path(DATA_DIR, fileName);
> resultFile = new Path(DATA_DIR, resultFileName);
> }else{
> System.out.println("Invalid command line option.");
> return;
> }
>
> FileStatusChecker checker = new TestDFSWrite().new
> FileStatusChecker(file, resultFile, interval);
> System.out.println(file.toString());
> //System.out.println("F L:" +
> newLongWritable(fs.getContentLength(file)).toString());
> out = fs.create(file);
> checker.start();
> long written = 0;
> try {
> finished = false;
> while (written < total) {
> long remains = total - written;
> int length = (remains<=buffer.length) ? (int)remains :
> buffer.length;
> out.write(buffer, 0, length);
> out.flush();
> written += length;
> System.out.println("One segment done!");
> System.out.println("F L:" + new
> LongWritable(fs.getFileStatus(file).getLen()).toString());
> }
> finished =true;
> } finally {
> out.close();
> }
> //System.exit(0);
> }
> }
>
> Neither the inner class FileStatusChecker nor the main function can get
> the right size, they both get "0".
>
> What should be the problem?
>
Re: Can't get DFS file size
Posted by "11 Nov." <no...@gmail.com>.
That's obviously impossible:)
2008/4/8, Michaela Buergle <Mi...@neofonie.de>:
>
> Maybe because you pass Strings to the LongWritables?
>
> micha
>
>
> 11 Nov. wrote:
> > Hi folks,
> > I'm writing a little test programm to check the writing speed of DFS
> > file system, but can't get the file size using
> > "fs.getFileStatus(file).getLen()" or fs.getContentLength(file). Here is
> my
> > code:
> >
> > package org.apache.hadoop.fs;
> > import java.io.IOException;
> > import java.io.OutputStream;
> >
> > import junit.framework.TestCase;
> >
> > import org.apache.hadoop.conf.Configuration;
> > import org.apache.hadoop.fs.FileSystem;
> > import org.apache.hadoop.io.LongWritable;
> > import org.apache.hadoop.io.SequenceFile;
> > import org.apache.hadoop.io.SequenceFile.CompressionType;
> > import org.apache.hadoop.io.Text;
> >
> > public class TestDFSWrite extends TestCase {
> > static String ROOT = System.getProperty("test.build.data","fs_test");
> > static Path DATA_DIR = new Path(ROOT, "fs_data");
> > static long MEGA = 1024 * 1024;
> > static int BUFFER_SIZE = 4096;
> > static Configuration conf = new Configuration();
> > static FileSystem fs;
> >
> > static byte[] buffer = new byte[BUFFER_SIZE];
> > static boolean finished;
> >
> > public class FileStatusChecker extends Thread{
> > Path file;
> > Path resultFile;
> > int interval;
> >
> > public FileStatusChecker(Path file, Path resultFile, int interval){
> > this.file = file;
> > this.resultFile = resultFile;
> > this.interval = interval;
> > }
> >
> > public void run(){
> > System.out.println("Here is the checker running!!");
> > System.out.println(file.toString());
> > SequenceFile.Writer writer = null;
> > try {
> > long lastLen = 1;
> > long thisLen = 1;
> > writer = SequenceFile.createWriter(fs, conf, resultFile,
> > Text.class, Text.class,
> > CompressionType.NONE);
> >
> > while(!finished){
> > lastLen = thisLen;
> > if(fs.exists(file)){
> > System.out.println("File exists!");
> > thisLen = fs.getContentLength(file);
> > System.out.println("@@"+thisLen);
> > }
> > else{
> > sleep(interval * 10);
> > continue;
> > }
> > long length = thisLen -lastLen;
> > System.out.println("thisLen is" + thisLen + "lastLen is " +
> > lastLen);
> > long cur = System.currentTimeMillis();
> > cur = cur - (cur % 10);
> > LongWritable current = new LongWritable(cur);
> > writer.append(new Text(current.toString()), new Text(new
> > LongWritable(length).toString()));
> > sleep(interval * 10);
> >
> > }
> > return;
> >
> > }catch (Exception e) {
> > e.printStackTrace();
> > }finally {
> > try {
> > writer.close();
> > } catch (Exception e) {
> > e.printStackTrace();
> > }
> > }
> > }
> > }
> >
> > public static void main(String[] args) throws Exception {
> > {
> > try {
> > fs = FileSystem.get(conf);
> > } catch (IOException e) {
> > throw new RuntimeException(e);
> > }
> > }
> > String testFunc = "";
> > String fileName = "";
> > String resultFileName = "";
> > int fileSize = 0;
> > int interval = 0;
> >
> > String usage = "Usage: TestDFSWrite -testfunc [read/write] -file foo
> > -size M -interval MS -result resultFile";
> > if (args.length == 0) {
> > System.err.println(usage);
> > System.exit(-1);
> > }
> >
> > for (int i = 0; i < args.length; i++) { // parse command line
> > if (args[i].equals("-testfunc")) {
> > testFunc = args[++i];
> > } else if (args[i].equals("-file")) {
> > fileName = args[++i];
> > } else if (args[i].equals("-size")) {
> > fileSize = Integer.parseInt(args[++i]);
> > } else if (args[i].equals("-interval")) {
> > interval = Integer.parseInt(args[++i]);
> > } else if (args[i].equals("-result")) {
> > resultFileName = args[++i];
> > }
> > }
> >
> > long total = fileSize * MEGA;
> > OutputStream out;
> > Path file, resultFile;
> > fs.delete(DATA_DIR);
> >
> > if(testFunc.equals("read")){
> > System.out.println("This option is not ready.");
> > return;
> > }else if(testFunc.equals("write")){
> > file = new Path(DATA_DIR, fileName);
> > resultFile = new Path(DATA_DIR, resultFileName);
> > }else{
> > System.out.println("Invalid command line option.");
> > return;
> > }
> >
> > FileStatusChecker checker = new TestDFSWrite().new
> > FileStatusChecker(file, resultFile, interval);
> > System.out.println(file.toString());
> > //System.out.println("F L:" +
> > newLongWritable(fs.getContentLength(file)).toString());
> > out = fs.create(file);
> > checker.start();
> > long written = 0;
> > try {
> > finished = false;
> > while (written < total) {
> > long remains = total - written;
> > int length = (remains<=buffer.length) ? (int)remains :
> > buffer.length;
> > out.write(buffer, 0, length);
> > out.flush();
> > written += length;
> > System.out.println("One segment done!");
> > System.out.println("F L:" + new
> > LongWritable(fs.getFileStatus(file).getLen()).toString());
> > }
> > finished =true;
> > } finally {
> > out.close();
> > }
> > //System.exit(0);
> > }
> > }
> >
> > Neither the inner class FileStatusChecker nor the main function can get
> the
> > right size, they both get "0".
> >
> > What should be the problem?
> >
>
>
> --
> --------------------------------
> Michaela Bürgle
> Softwareentwicklerin
> Forschung und Entwicklung
>
> neofonie GmbH
> Robert-Koch-Platz 4
> 10115 Berlin
> fon: +49.30 24627 256
> fax: +49.30 24627 120
> michaela.buergle@neofonie.de
> http://www.neofonie.de
>
> Handelsregister
> Berlin-Charlottenburg: HRB 67460
>
> Geschaeftsfuehrung
> Helmut Hoffer von Ankershoffen
> Nurhan Yildirim
> --------------------------------
>
Re: Can't get DFS file size
Posted by Michaela Buergle <Mi...@neofonie.de>.
Maybe because you pass Strings to the LongWritables?
micha
11 Nov. wrote:
> Hi folks,
> I'm writing a little test programm to check the writing speed of DFS
> file system, but can't get the file size using
> "fs.getFileStatus(file).getLen()" or fs.getContentLength(file). Here is my
> code:
>
> package org.apache.hadoop.fs;
> import java.io.IOException;
> import java.io.OutputStream;
>
> import junit.framework.TestCase;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.fs.FileSystem;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.SequenceFile;
> import org.apache.hadoop.io.SequenceFile.CompressionType;
> import org.apache.hadoop.io.Text;
>
> public class TestDFSWrite extends TestCase {
> static String ROOT = System.getProperty("test.build.data","fs_test");
> static Path DATA_DIR = new Path(ROOT, "fs_data");
> static long MEGA = 1024 * 1024;
> static int BUFFER_SIZE = 4096;
> static Configuration conf = new Configuration();
> static FileSystem fs;
>
> static byte[] buffer = new byte[BUFFER_SIZE];
> static boolean finished;
>
> public class FileStatusChecker extends Thread{
> Path file;
> Path resultFile;
> int interval;
>
> public FileStatusChecker(Path file, Path resultFile, int interval){
> this.file = file;
> this.resultFile = resultFile;
> this.interval = interval;
> }
>
> public void run(){
> System.out.println("Here is the checker running!!");
> System.out.println(file.toString());
> SequenceFile.Writer writer = null;
> try {
> long lastLen = 1;
> long thisLen = 1;
> writer = SequenceFile.createWriter(fs, conf, resultFile,
> Text.class, Text.class,
> CompressionType.NONE);
>
> while(!finished){
> lastLen = thisLen;
> if(fs.exists(file)){
> System.out.println("File exists!");
> thisLen = fs.getContentLength(file);
> System.out.println("@@"+thisLen);
> }
> else{
> sleep(interval * 10);
> continue;
> }
> long length = thisLen -lastLen;
> System.out.println("thisLen is" + thisLen + "lastLen is " +
> lastLen);
> long cur = System.currentTimeMillis();
> cur = cur - (cur % 10);
> LongWritable current = new LongWritable(cur);
> writer.append(new Text(current.toString()), new Text(new
> LongWritable(length).toString()));
> sleep(interval * 10);
>
> }
> return;
>
> }catch (Exception e) {
> e.printStackTrace();
> }finally {
> try {
> writer.close();
> } catch (Exception e) {
> e.printStackTrace();
> }
> }
> }
> }
>
> public static void main(String[] args) throws Exception {
> {
> try {
> fs = FileSystem.get(conf);
> } catch (IOException e) {
> throw new RuntimeException(e);
> }
> }
> String testFunc = "";
> String fileName = "";
> String resultFileName = "";
> int fileSize = 0;
> int interval = 0;
>
> String usage = "Usage: TestDFSWrite -testfunc [read/write] -file foo
> -size M -interval MS -result resultFile";
> if (args.length == 0) {
> System.err.println(usage);
> System.exit(-1);
> }
>
> for (int i = 0; i < args.length; i++) { // parse command line
> if (args[i].equals("-testfunc")) {
> testFunc = args[++i];
> } else if (args[i].equals("-file")) {
> fileName = args[++i];
> } else if (args[i].equals("-size")) {
> fileSize = Integer.parseInt(args[++i]);
> } else if (args[i].equals("-interval")) {
> interval = Integer.parseInt(args[++i]);
> } else if (args[i].equals("-result")) {
> resultFileName = args[++i];
> }
> }
>
> long total = fileSize * MEGA;
> OutputStream out;
> Path file, resultFile;
> fs.delete(DATA_DIR);
>
> if(testFunc.equals("read")){
> System.out.println("This option is not ready.");
> return;
> }else if(testFunc.equals("write")){
> file = new Path(DATA_DIR, fileName);
> resultFile = new Path(DATA_DIR, resultFileName);
> }else{
> System.out.println("Invalid command line option.");
> return;
> }
>
> FileStatusChecker checker = new TestDFSWrite().new
> FileStatusChecker(file, resultFile, interval);
> System.out.println(file.toString());
> //System.out.println("F L:" +
> newLongWritable(fs.getContentLength(file)).toString());
> out = fs.create(file);
> checker.start();
> long written = 0;
> try {
> finished = false;
> while (written < total) {
> long remains = total - written;
> int length = (remains<=buffer.length) ? (int)remains :
> buffer.length;
> out.write(buffer, 0, length);
> out.flush();
> written += length;
> System.out.println("One segment done!");
> System.out.println("F L:" + new
> LongWritable(fs.getFileStatus(file).getLen()).toString());
> }
> finished =true;
> } finally {
> out.close();
> }
> //System.exit(0);
> }
> }
>
> Neither the inner class FileStatusChecker nor the main function can get the
> right size, they both get "0".
>
> What should be the problem?
>
--
--------------------------------
Michaela Bürgle
Softwareentwicklerin
Forschung und Entwicklung
neofonie GmbH
Robert-Koch-Platz 4
10115 Berlin
fon: +49.30 24627 256
fax: +49.30 24627 120
michaela.buergle@neofonie.de
http://www.neofonie.de
Handelsregister
Berlin-Charlottenburg: HRB 67460
Geschaeftsfuehrung
Helmut Hoffer von Ankershoffen
Nurhan Yildirim
--------------------------------