You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-dev@hadoop.apache.org by "Jason Lowe (JIRA)" <ji...@apache.org> on 2013/12/04 15:36:35 UTC
[jira] [Resolved] (MAPREDUCE-5666)
org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):cannot
find symbol
[ https://issues.apache.org/jira/browse/MAPREDUCE-5666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jason Lowe resolved MAPREDUCE-5666.
-----------------------------------
Resolution: Invalid
As I mentioned in MAPREDUCE-5668, JIRA is not the avenue to use for questions like this. Please use the [user@ mailing list|http://hadoop.apache.org/mailing_lists.html#User] for questions like this. If after discussing on the mailing list it ends up being a bug in Hadoop then a JIRA can be filed at that time.
This is clearly a case of code being compiled against a release after 1.x but then run on a 0.20 or 1.x release. The FileStatus.isDirectory() method was not present in the 0.20 or 1.x releases. We cannot generally support compiling code against a later release and then running it on an earlier release because of new APIs that can be added.
> org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):cannot find symbol
> -----------------------------------------------------------------------------------------------------------------------------------------
>
> Key: MAPREDUCE-5666
> URL: https://issues.apache.org/jira/browse/MAPREDUCE-5666
> Project: Hadoop Map/Reduce
> Issue Type: Bug
> Reporter: ranjini
>
> hi
> I have written the below code , and facing the issue. i am using hadoop 0.20 vesion and java 1.6 the issue is
> org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):232: cannot find symbol
> symbol : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
> if (globStat.isDirectory()) {
> ^
> org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java(org/apache/hadoop/mapreduce/lib/output:FileOutputCommitter.java):208: cannot find symbol
> symbol : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
> } else if(fs.getFileStatus(taskOutput).isDirectory()) {
> ^
> org/apache/hadoop/mapred/JobConf.java(org/apache/hadoop/mapred:JobConf.java):433: cannot find symbol
> symbol : method getPattern(java.lang.String,java.util.regex.Pattern)
> location: class org.apache.hadoop.mapred.JobConf
> return getPattern(JobContext.JAR_UNPACK_PATTERN, UNPACK_JAR_PATTERN_DEFAULT);
> ^
> org/apache/hadoop/mapred/JobConf.java(org/apache/hadoop/mapred:JobConf.java):450: cannot find symbol
> symbol : method getTrimmedStrings(java.lang.String)
> location: class org.apache.hadoop.mapred.JobConf
> return getTrimmedStrings(MRConfig.LOCAL_DIR);
> ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):165: cannot find symbol
> symbol : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
> if (stat.isDirectory()) {
> ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):215: cannot find symbol
> symbol : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
> if (globStat.isDirectory()) {
> ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):218: cannot find symbol
> symbol : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
> if (recursive && stat.isDirectory()) {
> ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):258: cannot find symbol
> symbol : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
> if (file.isDirectory()) {
> ^
> org/apache/hadoop/mapred/FileOutputCommitter.java(org/apache/hadoop/mapred:FileOutputCommitter.java):166: cannot find symbol
> symbol : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
> } else if(fs.getFileStatus(taskOutput).isDirectory()) {
> ^
> org/apache/hadoop/mapred/LineRecordReader.java(org/apache/hadoop/mapred:LineRecordReader.java):100: incompatible types
> found : org.apache.hadoop.io.compress.SplitCompressionInputStream
> required: org.apache.hadoop.fs.Seekable
> filePosition = cIn; // take pos from compressed stream
> ^
> org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java(org/apache/hadoop/mapreduce/lib/input:LineRecordReader.java):98: incompatible types
> found : org.apache.hadoop.io.compress.SplitCompressionInputStream
> required: org.apache.hadoop.fs.Seekable
> filePosition = cIn;
> I have attached the code
> import java.io.DataInput;
> import java.io.DataOutput;
> import java.io.IOException;
> import java.util.StringTokenizer;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.FSDataInputStream;
> import org.apache.hadoop.fs.FileSystem;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.io.WritableComparable;
> import org.apache.hadoop.mapreduce.InputSplit;
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
> import org.apache.hadoop.mapreduce.RecordReader;
> import org.apache.hadoop.mapreduce.TaskAttemptContext;
> import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;
> import org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader;
> import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
> import org.apache.hadoop.util.LineReader;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
> /**
> * MultiFileWordCount is an example to demonstrate the usage of
> * MultiFileInputFormat. This examples counts the occurrences of
> * words in the text files under the given input directory.
> */
> public class MultiFileWordCount extends Configured implements Tool {
> /**
> * This record keeps <filename,offset> pairs.
> */
> public static class WordOffset implements WritableComparable {
> private long offset;
> private String fileName;
>
> public void readFields(DataInput in) throws IOException {
> this.offset = in.readLong();
> this.fileName = Text.readString(in);
> }
> public void write(DataOutput out) throws IOException {
> out.writeLong(offset);
> Text.writeString(out, fileName);
> }
> public int compareTo(Object o) {
> WordOffset that = (WordOffset)o;
> int f = this.fileName.compareTo(that.fileName);
> if(f == 0) {
> return (int)Math.signum((double)(this.offset - that.offset));
> }
> return f;
> }
> @Override
> public boolean equals(Object obj) {
> if(obj instanceof WordOffset)
> return this.compareTo(obj) == 0;
> return false;
> }
> @Override
> public int hashCode() {
> assert false : "hashCode not designed";
> return 42; //an arbitrary constant
> }
> }
> /**
> * To use {@link CombineFileInputFormat}, one should extend it, to return a
> * (custom) {@link RecordReader}. CombineFileInputFormat uses
> * {@link CombineFileSplit}s.
> */
> public static class MyInputFormat
> extends CombineFileInputFormat {
> public RecordReader createRecordReader(InputSplit split,
> TaskAttemptContext context) throws IOException {
> return new CombineFileRecordReader(
> (CombineFileSplit)split, context, CombineFileLineRecordReader.class);
> }
> }
>
> /**
> * RecordReader is responsible from extracting records from a chunk
> * of the CombineFileSplit.
> */
> public static class CombineFileLineRecordReader
> extends RecordReader {
> private long startOffset; //offset of the chunk;
> private long end; //end of the chunk;
> private long pos; // current pos
> private FileSystem fs;
> private Path path;
> private WordOffset key;
> private Text value;
> private FSDataInputStream fileIn;
> private LineReader reader;
> public CombineFileLineRecordReader(CombineFileSplit split,
> TaskAttemptContext context, Integer index) throws IOException {
>
> this.path = split.getPath(index);
> fs = this.path.getFileSystem(context.getConfiguration());
> this.startOffset = split.getOffset(index);
> this.end = startOffset + split.getLength(index);
> boolean skipFirstLine = false;
> //open the file
> fileIn = fs.open(path);
> if (startOffset != 0) {
> skipFirstLine = true;
> --startOffset;
> fileIn.seek(startOffset);
> }
> reader = new LineReader(fileIn);
> if (skipFirstLine) { // skip first line and re-establish "startOffset".
> startOffset += reader.readLine(new Text(), 0,
> (int)Math.min((long)Integer.MAX_VALUE, end - startOffset));
> }
> this.pos = startOffset;
> }
> public void initialize(InputSplit split, TaskAttemptContext context)
> throws IOException, InterruptedException {
> }
> public void close() throws IOException { }
> public float getProgress() throws IOException {
> if (startOffset == end) {
> return 0.0f;
> } else {
> return Math.min(1.0f, (pos - startOffset) / (float)(end - startOffset));
> }
> }
> public boolean nextKeyValue() throws IOException {
> if (key == null) {
> key = new WordOffset();
> key.fileName = path.getName();
> }
> key.offset = pos;
> if (value == null) {
> value = new Text();
> }
> int newSize = 0;
> if (pos < end) {
> newSize = reader.readLine(value);
> pos += newSize;
> }
> if (newSize == 0) {
> key = null;
> value = null;
> return false;
> } else {
> return true;
> }
> }
> public WordOffset getCurrentKey()
> throws IOException, InterruptedException {
> return key;
> }
> public Text getCurrentValue() throws IOException, InterruptedException {
> return value;
> }
> }
>
> /**
> * This Mapper is similar to the one in {@link WordCount.MapClass}.
> */
> public static class MapClass extends
> Mapper {
> private final static IntWritable one = new IntWritable(1);
> private Text word = new Text();
> public void map(WordOffset key, Text value, Context context)
> throws IOException, InterruptedException {
> String line = value.toString();
> StringTokenizer itr = new StringTokenizer(line);
> while (itr.hasMoreTokens()) {
> word.set(itr.nextToken());
> context.write(word, one);
> }
> }
> }
> private void printUsage() {
> System.out.println("Usage : multifilewc " );
> }
> public int run(String[] args) throws Exception {
> if(args.length < 2) {
> printUsage();
> return 2;
> }
> Job job = new Job(getConf());
> job.setJobName("MultiFileWordCount");
> job.setJarByClass(MultiFileWordCount.class);
>
> //set the InputFormat of the job to our InputFormat
> job.setInputFormatClass(MyInputFormat.class);
>
> // the keys are words (strings)
> job.setOutputKeyClass(Text.class);
> // the values are counts (ints)
> job.setOutputValueClass(IntWritable.class);
>
> //use the defined mapper
> job.setMapperClass(MapClass.class);
> //use the WordCount Reducer
> job.setCombinerClass(IntSumReducer.class);
> job.setReducerClass(IntSumReducer.class);
>
> FileInputFormat.addInputPaths(job, args[0]);
> FileOutputFormat.setOutputPath(job, new Path(args[1]));
> return job.waitForCompletion(true) ? 0 : 1;
> }
>
> public static void main(String[] args) throws Exception {
> int ret = ToolRunner.run(new MultiFileWordCount(), args);
> System.exit(ret);
> }
> }
--
This message was sent by Atlassian JIRA
(v6.1#6144)