You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-user@hadoop.apache.org by jamal sasha <ja...@gmail.com> on 2013/08/27 20:39:38 UTC
libjars error
I have bunch of jars whcih i want to pass. I am using libjars option to do
so. But to do that I have to implement tool ??
So i change my code to following but still I am getting this warning?
13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
parsing the arguments. Applications should implement Tool for the same.
and it is not finding that jar (in bold??)
package org.foobar;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
*import com.hp.hpl.jena.rdf.model.*;*
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class RdfFormatter extends Configured implements Tool {
//foo bar
}
public int run(String [] args) throws Exception{
//Configuration conf = new Configuration();
Job job = new Job();
//job.setMapOutputKeyClass(Pair.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
//conf.set("delimiter", "\\s+");
job.setJarByClass(RdfFormatter.class);
job.setMapperClass(RDFMapper.class);
//job.setReducerClass(MeanReducer.class);
job.setNumReduceTasks(0);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new
Path("/user/user/mohit/wiki-rdf/sample-input"));
FileOutputFormat.setOutputPath(job, new
Path("/user/user/mohit/wiki-rdf/outputter"));
job.submit();
long start = new Date().getTime();
job.waitForCompletion(true);
long end = new Date().getTime();
System.out.println("Job took " + ((end-start)/1000) + " seconds" );
return 0;
}
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
/*
Configuration conf = new Configuration();
Job job = new Job(conf, "RDF input format");
//job.setMapOutputKeyClass(Pair.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
conf.set("delimiter", "\\s+");
job.setJarByClass(Formatter.class);
job.setMapperClass(Mapper.class);
//job.setReducerClass(Reducer.class);
job.setNumReduceTasks(0);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path("/input/data"));
FileOutputFormat.setOutputPath(job, new Path("/output"));
// set timer
*
*/
Configuration conf = new Configuration();
int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
run() method.
System.exit(ret);
}
}
Re: libjars error
Posted by Harsh J <ha...@cloudera.com>.
You're not using the configuration provided by the ToolRunner instantiation:
Change:
Job job = new Job();
To:
Job job = new Job(getConf());
On Wed, Aug 28, 2013 at 12:09 AM, jamal sasha <ja...@gmail.com> wrote:
> I have bunch of jars whcih i want to pass. I am using libjars option to do
> so. But to do that I have to implement tool ??
> So i change my code to following but still I am getting this warning?
> 13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
> parsing the arguments. Applications should implement Tool for the same.
>
> and it is not finding that jar (in bold??)
> package org.foobar;
>
> import java.io.ByteArrayInputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.NullWritable;
> import org.apache.hadoop.io.Text;
> import com.hp.hpl.jena.rdf.model.*;
>
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
>
>
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
>
> public class RdfFormatter extends Configured implements Tool {
> //foo bar
> }
> public int run(String [] args) throws Exception{
> //Configuration conf = new Configuration();
>
> Job job = new Job();
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(Text.class);
> //conf.set("delimiter", "\\s+");
>
> job.setJarByClass(RdfFormatter.class);
> job.setMapperClass(RDFMapper.class);
> //job.setReducerClass(MeanReducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new
> Path("/user/user/mohit/wiki-rdf/sample-input"));
> FileOutputFormat.setOutputPath(job, new
> Path("/user/user/mohit/wiki-rdf/outputter"));
> job.submit();
> long start = new Date().getTime();
> job.waitForCompletion(true);
> long end = new Date().getTime();
> System.out.println("Job took " + ((end-start)/1000) + " seconds" );
> return 0;
> }
> public static void main(String[] args) throws Exception {
> // TODO Auto-generated method stub
> /*
> Configuration conf = new Configuration();
>
> Job job = new Job(conf, "RDF input format");
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(IntWritable.class);
> conf.set("delimiter", "\\s+");
>
> job.setJarByClass(Formatter.class);
> job.setMapperClass(Mapper.class);
> //job.setReducerClass(Reducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new Path("/input/data"));
> FileOutputFormat.setOutputPath(job, new Path("/output"));
> // set timer
> *
> */
> Configuration conf = new Configuration();
> int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
> run() method.
> System.exit(ret);
>
>
>
>
>
> }
>
> }
>
--
Harsh J
Re: libjars error
Posted by Harsh J <ha...@cloudera.com>.
You're not using the configuration provided by the ToolRunner instantiation:
Change:
Job job = new Job();
To:
Job job = new Job(getConf());
On Wed, Aug 28, 2013 at 12:09 AM, jamal sasha <ja...@gmail.com> wrote:
> I have bunch of jars whcih i want to pass. I am using libjars option to do
> so. But to do that I have to implement tool ??
> So i change my code to following but still I am getting this warning?
> 13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
> parsing the arguments. Applications should implement Tool for the same.
>
> and it is not finding that jar (in bold??)
> package org.foobar;
>
> import java.io.ByteArrayInputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.NullWritable;
> import org.apache.hadoop.io.Text;
> import com.hp.hpl.jena.rdf.model.*;
>
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
>
>
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
>
> public class RdfFormatter extends Configured implements Tool {
> //foo bar
> }
> public int run(String [] args) throws Exception{
> //Configuration conf = new Configuration();
>
> Job job = new Job();
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(Text.class);
> //conf.set("delimiter", "\\s+");
>
> job.setJarByClass(RdfFormatter.class);
> job.setMapperClass(RDFMapper.class);
> //job.setReducerClass(MeanReducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new
> Path("/user/user/mohit/wiki-rdf/sample-input"));
> FileOutputFormat.setOutputPath(job, new
> Path("/user/user/mohit/wiki-rdf/outputter"));
> job.submit();
> long start = new Date().getTime();
> job.waitForCompletion(true);
> long end = new Date().getTime();
> System.out.println("Job took " + ((end-start)/1000) + " seconds" );
> return 0;
> }
> public static void main(String[] args) throws Exception {
> // TODO Auto-generated method stub
> /*
> Configuration conf = new Configuration();
>
> Job job = new Job(conf, "RDF input format");
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(IntWritable.class);
> conf.set("delimiter", "\\s+");
>
> job.setJarByClass(Formatter.class);
> job.setMapperClass(Mapper.class);
> //job.setReducerClass(Reducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new Path("/input/data"));
> FileOutputFormat.setOutputPath(job, new Path("/output"));
> // set timer
> *
> */
> Configuration conf = new Configuration();
> int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
> run() method.
> System.exit(ret);
>
>
>
>
>
> }
>
> }
>
--
Harsh J
Re: libjars error
Posted by Harsh J <ha...@cloudera.com>.
You're not using the configuration provided by the ToolRunner instantiation:
Change:
Job job = new Job();
To:
Job job = new Job(getConf());
On Wed, Aug 28, 2013 at 12:09 AM, jamal sasha <ja...@gmail.com> wrote:
> I have bunch of jars whcih i want to pass. I am using libjars option to do
> so. But to do that I have to implement tool ??
> So i change my code to following but still I am getting this warning?
> 13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
> parsing the arguments. Applications should implement Tool for the same.
>
> and it is not finding that jar (in bold??)
> package org.foobar;
>
> import java.io.ByteArrayInputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.NullWritable;
> import org.apache.hadoop.io.Text;
> import com.hp.hpl.jena.rdf.model.*;
>
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
>
>
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
>
> public class RdfFormatter extends Configured implements Tool {
> //foo bar
> }
> public int run(String [] args) throws Exception{
> //Configuration conf = new Configuration();
>
> Job job = new Job();
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(Text.class);
> //conf.set("delimiter", "\\s+");
>
> job.setJarByClass(RdfFormatter.class);
> job.setMapperClass(RDFMapper.class);
> //job.setReducerClass(MeanReducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new
> Path("/user/user/mohit/wiki-rdf/sample-input"));
> FileOutputFormat.setOutputPath(job, new
> Path("/user/user/mohit/wiki-rdf/outputter"));
> job.submit();
> long start = new Date().getTime();
> job.waitForCompletion(true);
> long end = new Date().getTime();
> System.out.println("Job took " + ((end-start)/1000) + " seconds" );
> return 0;
> }
> public static void main(String[] args) throws Exception {
> // TODO Auto-generated method stub
> /*
> Configuration conf = new Configuration();
>
> Job job = new Job(conf, "RDF input format");
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(IntWritable.class);
> conf.set("delimiter", "\\s+");
>
> job.setJarByClass(Formatter.class);
> job.setMapperClass(Mapper.class);
> //job.setReducerClass(Reducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new Path("/input/data"));
> FileOutputFormat.setOutputPath(job, new Path("/output"));
> // set timer
> *
> */
> Configuration conf = new Configuration();
> int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
> run() method.
> System.exit(ret);
>
>
>
>
>
> }
>
> }
>
--
Harsh J
Re: libjars error
Posted by Harsh J <ha...@cloudera.com>.
You're not using the configuration provided by the ToolRunner instantiation:
Change:
Job job = new Job();
To:
Job job = new Job(getConf());
On Wed, Aug 28, 2013 at 12:09 AM, jamal sasha <ja...@gmail.com> wrote:
> I have bunch of jars whcih i want to pass. I am using libjars option to do
> so. But to do that I have to implement tool ??
> So i change my code to following but still I am getting this warning?
> 13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
> parsing the arguments. Applications should implement Tool for the same.
>
> and it is not finding that jar (in bold??)
> package org.foobar;
>
> import java.io.ByteArrayInputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.NullWritable;
> import org.apache.hadoop.io.Text;
> import com.hp.hpl.jena.rdf.model.*;
>
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
>
>
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
>
> public class RdfFormatter extends Configured implements Tool {
> //foo bar
> }
> public int run(String [] args) throws Exception{
> //Configuration conf = new Configuration();
>
> Job job = new Job();
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(Text.class);
> //conf.set("delimiter", "\\s+");
>
> job.setJarByClass(RdfFormatter.class);
> job.setMapperClass(RDFMapper.class);
> //job.setReducerClass(MeanReducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new
> Path("/user/user/mohit/wiki-rdf/sample-input"));
> FileOutputFormat.setOutputPath(job, new
> Path("/user/user/mohit/wiki-rdf/outputter"));
> job.submit();
> long start = new Date().getTime();
> job.waitForCompletion(true);
> long end = new Date().getTime();
> System.out.println("Job took " + ((end-start)/1000) + " seconds" );
> return 0;
> }
> public static void main(String[] args) throws Exception {
> // TODO Auto-generated method stub
> /*
> Configuration conf = new Configuration();
>
> Job job = new Job(conf, "RDF input format");
> //job.setMapOutputKeyClass(Pair.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(IntWritable.class);
> conf.set("delimiter", "\\s+");
>
> job.setJarByClass(Formatter.class);
> job.setMapperClass(Mapper.class);
> //job.setReducerClass(Reducer.class);
> job.setNumReduceTasks(0);
>
> job.setInputFormatClass(TextInputFormat.class);
> job.setOutputFormatClass(TextOutputFormat.class);
>
> FileInputFormat.addInputPath(job, new Path("/input/data"));
> FileOutputFormat.setOutputPath(job, new Path("/output"));
> // set timer
> *
> */
> Configuration conf = new Configuration();
> int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
> run() method.
> System.exit(ret);
>
>
>
>
>
> }
>
> }
>
--
Harsh J