You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-issues@hadoop.apache.org by "Arsen Zahray (JIRA)" <ji...@apache.org> on 2011/09/11 11:31:08 UTC
[jira] [Updated] (HADOOP-7622) Packaging libraries into generated
JAR causes ClassNotFoundException on Clusters
[ https://issues.apache.org/jira/browse/HADOOP-7622?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Arsen Zahray updated HADOOP-7622:
---------------------------------
Description:
When writing an app, if you package the code in eclipse with "Package libraries into generated JAR" instead of "Extract generated libraries into generated JAR", the job will fail with "java.lang.RuntimeException: java.lang.ClassNotFoundException".
Steps to reproduce:
1. Take heloworld app(word counter):
package com.iathao;
//package org.myorg;
import java.io.IOException;
import java.util.Iterator;
import java.util.StringTokenizer;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
public class WordCount {
public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
String line = value.toString();
StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
word.set(tokenizer.nextToken());
output.collect(word, one);
}
}
}
public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
int sum = 0;
while (values.hasNext()) {
sum += values.next().get();
}
output.collect(key, new IntWritable(sum));
}
}
public static void main(String[] args) throws Exception {
JobConf conf = new JobConf(WordCount.class);
conf.setJobName("wordcount");
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(Map.class);
conf.setCombinerClass(Reduce.class);
conf.setReducerClass(Reduce.class);
conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(TextOutputFormat.class);
FileInputFormat.setInputPaths(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
JobClient.runJob(conf);
}
}
2. In Eclipse, package go to Export->Runneable Jar->Package libraries into generated JAR
3. Run the job on Amazon Elastic Mapreduce
Environment: Elastic Mapreduce
Summary: Packaging libraries into generated JAR causes ClassNotFoundException on Clusters (was: Packaging libraries into generated JAR causes Class)
> Packaging libraries into generated JAR causes ClassNotFoundException on Clusters
> --------------------------------------------------------------------------------
>
> Key: HADOOP-7622
> URL: https://issues.apache.org/jira/browse/HADOOP-7622
> Project: Hadoop Common
> Issue Type: Bug
> Environment: Elastic Mapreduce
> Reporter: Arsen Zahray
> Priority: Minor
>
> When writing an app, if you package the code in eclipse with "Package libraries into generated JAR" instead of "Extract generated libraries into generated JAR", the job will fail with "java.lang.RuntimeException: java.lang.ClassNotFoundException".
> Steps to reproduce:
> 1. Take heloworld app(word counter):
> package com.iathao;
> //package org.myorg;
> import java.io.IOException;
> import java.util.Iterator;
> import java.util.StringTokenizer;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapred.FileInputFormat;
> import org.apache.hadoop.mapred.FileOutputFormat;
> import org.apache.hadoop.mapred.JobClient;
> import org.apache.hadoop.mapred.JobConf;
> import org.apache.hadoop.mapred.MapReduceBase;
> import org.apache.hadoop.mapred.Mapper;
> import org.apache.hadoop.mapred.OutputCollector;
> import org.apache.hadoop.mapred.Reducer;
> import org.apache.hadoop.mapred.Reporter;
> import org.apache.hadoop.mapred.TextInputFormat;
> import org.apache.hadoop.mapred.TextOutputFormat;
> public class WordCount {
> public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
> private final static IntWritable one = new IntWritable(1);
> private Text word = new Text();
> public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
> String line = value.toString();
> StringTokenizer tokenizer = new StringTokenizer(line);
> while (tokenizer.hasMoreTokens()) {
> word.set(tokenizer.nextToken());
> output.collect(word, one);
> }
> }
> }
> public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
> public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
> int sum = 0;
> while (values.hasNext()) {
> sum += values.next().get();
> }
> output.collect(key, new IntWritable(sum));
> }
> }
> public static void main(String[] args) throws Exception {
> JobConf conf = new JobConf(WordCount.class);
> conf.setJobName("wordcount");
> conf.setOutputKeyClass(Text.class);
> conf.setOutputValueClass(IntWritable.class);
> conf.setMapperClass(Map.class);
> conf.setCombinerClass(Reduce.class);
> conf.setReducerClass(Reduce.class);
> conf.setInputFormat(TextInputFormat.class);
> conf.setOutputFormat(TextOutputFormat.class);
> FileInputFormat.setInputPaths(conf, new Path(args[0]));
> FileOutputFormat.setOutputPath(conf, new Path(args[1]));
> JobClient.runJob(conf);
> }
> }
> 2. In Eclipse, package go to Export->Runneable Jar->Package libraries into generated JAR
> 3. Run the job on Amazon Elastic Mapreduce
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira