You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hadoop.apache.org by Omkar Joshi <Om...@lntinfotech.com> on 2013/06/13 06:22:44 UTC
Reducer not getting called
Hi,
I have a SequenceFile which contains several jpeg images with (image name, image bytes) as key-value pairs. My objective is to count the no. of images by grouping them by the source, something like this :
Nikon Coolpix 100
Sony Cybershot 251
N82 100
The MR code is :
package com.hadoop.basics;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import com.drew.imaging.ImageMetadataReader;
import com.drew.imaging.ImageProcessingException;
import com.drew.metadata.Directory;
import com.drew.metadata.Metadata;
import com.drew.metadata.exif.ExifIFD0Directory;
public class ImageSummary extends Configured implements Tool {
public static class ImageSourceMapper extends
Mapper<Text, BytesWritable, Text, IntWritable> {
private static int tagId = 272;
private static final IntWritable one = new IntWritable(1);
public void map(Text imageName, BytesWritable imageBytes,
Context context) throws IOException, InterruptedException {
// TODO Auto-generated method stub
System.out.println("In the map method, image is "
+ imageName.toString());
byte[] imageInBytes = imageBytes.getBytes();
ByteArrayInputStream bais = new ByteArrayInputStream(imageInBytes);
BufferedInputStream bis = new BufferedInputStream(bais);
Metadata imageMD = null;
try {
imageMD = ImageMetadataReader.readMetadata(bis, true);
} catch (ImageProcessingException e) {
// TODO Auto-generated catch block
System.out.println("Got an ImageProcessingException !");
e.printStackTrace();
}
Directory exifIFD0Directory = imageMD
.getDirectory(ExifIFD0Directory.class);
String imageSource = exifIFD0Directory.getString(tagId);
System.out.println(imageName.toString() + " is taken using "
+ imageSource);
context.write(new Text(imageSource), one);
System.out.println("Returning from the map method");
}
}
public static class ImageSourceReducer extends
Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce(Text imageSource, Iterator<IntWritable> counts,
Context context) throws IOException, InterruptedException {
// TODO Auto-generated method stub
System.out.println("In the reduce method");
int finalCount = 0;
while (counts.hasNext()) {
finalCount += counts.next().get();
}
context.write(imageSource, new IntWritable(finalCount));
System.out.println("Returning from the reduce method");
}
}
public static void main(String[] args) throws Exception {
ToolRunner.run(new ImageSummary(), args);
}
@Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
System.out.println("In ImageSummary.run(...)");
Configuration configuration = getConf();
Job job = new Job(configuration, "Image_Source");
job.setJarByClass(getClass());
job.setInputFormatClass(SequenceFileInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setMapperClass(ImageSourceMapper.class);
job.setCombinerClass(ImageSourceReducer.class);
job.setReducerClass(ImageSourceReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
SequenceFileInputFormat.addInputPath(job, new Path(args[0]));
TextOutputFormat.setOutputPath(job, new Path(args[1]));
System.out.println("Submitting job");
job.waitForCompletion(true);
int jobStatus = job.isSuccessful() ? 0 : -1;
System.out.println("Returning jobStatus = " + jobStatus);
return jobStatus;
}
}
The command :
hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile" "/scratchpad/imageOps/cnt"
The part-file (/<http://172.25.6.71:50075/browseDirectory.jsp?dir=/&namenodeInfoPort=50070>scratchpad<http://172.25.6.71:50075/browseDirectory.jsp?dir=/scratchpad&namenodeInfoPort=50070>/imageOps<http://172.25.6.71:50075/browseDirectory.jsp?dir=/scratchpad/imageOps&namenodeInfoPort=50070>/cnt<http://172.25.6.71:50075/browseDirectory.jsp?dir=/scratchpad/imageOps/cnt&namenodeInfoPort=50070>/part-r-00000)
COOLPIX L120 1
COOLPIX L120 1
K750i 1
The mapper stdout logs :
stdout logs
In the map method, image is It's a long road....JPG
It's a long road....JPG is taken using COOLPIX L120
Returning from the map method
In the map method, image is Every man is a mountainside....JPG
Every man is a mountainside....JPG is taken using COOLPIX L120
Returning from the map method
In the map method, image is mystic.JPG
mystic.JPG is taken using K750i
Returning from the map method
But nothing is reflected in stdout logs of the reducer.
What have I missed?
Regards,
Omkar Joshi
________________________________
The contents of this e-mail and any attachment(s) may contain confidential or privileged information for the intended recipient(s). Unintended recipients are prohibited from taking action on the basis of information in this e-mail and using or disseminating the information, and must notify the sender and delete it from their system. L&T Infotech will not accept responsibility or liability for the accuracy or completeness of, or the presence of any virus or disabling code in this e-mail"
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Yeah, a silly mistake on my part – read and compared the classes in the API doc. And realized it.
Thanks !
Regards,
Omkar Joshi
From: Rahul Bhattacharjee [mailto:rahul.rec.dgp@gmail.com]
Sent: Thursday, June 13, 2013 6:38 PM
To: user@hadoop.apache.org
Subject: Re: Reducer not getting called
The programming error is already mentioned. You are actually not overriding base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi <Om...@lntinfotech.com>> wrote:
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com<ma...@cloudera.com>]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com>> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Yeah, a silly mistake on my part – read and compared the classes in the API doc. And realized it.
Thanks !
Regards,
Omkar Joshi
From: Rahul Bhattacharjee [mailto:rahul.rec.dgp@gmail.com]
Sent: Thursday, June 13, 2013 6:38 PM
To: user@hadoop.apache.org
Subject: Re: Reducer not getting called
The programming error is already mentioned. You are actually not overriding base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi <Om...@lntinfotech.com>> wrote:
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com<ma...@cloudera.com>]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com>> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Yeah, a silly mistake on my part – read and compared the classes in the API doc. And realized it.
Thanks !
Regards,
Omkar Joshi
From: Rahul Bhattacharjee [mailto:rahul.rec.dgp@gmail.com]
Sent: Thursday, June 13, 2013 6:38 PM
To: user@hadoop.apache.org
Subject: Re: Reducer not getting called
The programming error is already mentioned. You are actually not overriding base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi <Om...@lntinfotech.com>> wrote:
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com<ma...@cloudera.com>]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com>> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Yeah, a silly mistake on my part – read and compared the classes in the API doc. And realized it.
Thanks !
Regards,
Omkar Joshi
From: Rahul Bhattacharjee [mailto:rahul.rec.dgp@gmail.com]
Sent: Thursday, June 13, 2013 6:38 PM
To: user@hadoop.apache.org
Subject: Re: Reducer not getting called
The programming error is already mentioned. You are actually not overriding base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi <Om...@lntinfotech.com>> wrote:
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com<ma...@cloudera.com>]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com>> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
Re: Reducer not getting called
Posted by Rahul Bhattacharjee <ra...@gmail.com>.
The programming error is already mentioned. You are actually not overriding
base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi
<Om...@lntinfotech.com>wrote:
> Ok but that link is broken - can you provide a working one?
>
> Regards,
> Omkar Joshi
>
>
> -----Original Message-----
> From: Harsh J [mailto:harsh@cloudera.com]
> Sent: Thursday, June 13, 2013 11:01 AM
> To: <us...@hadoop.apache.org>
> Subject: Re: Reducer not getting called
>
> You're not using the recommended @Override annotations, and are
> hitting a classic programming mistake. Your issue is same as this
> earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
> ones before it).
>
> On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
> <Om...@lntinfotech.com> wrote:
> > Hi,
> >
> >
> >
> > I have a SequenceFile which contains several jpeg images with (image
> name,
> > image bytes) as key-value pairs. My objective is to count the no. of
> images
> > by grouping them by the source, something like this :
> >
> >
> >
> > Nikon Coolpix 100
> >
> > Sony Cybershot 251
> >
> > N82 100
> >
> >
> >
> >
> >
> > The MR code is :
> >
> >
> >
> > package com.hadoop.basics;
> >
> >
> >
> > import java.io.BufferedInputStream;
> >
> > import java.io.ByteArrayInputStream;
> >
> > import java.io.IOException;
> >
> > import java.util.Iterator;
> >
> >
> >
> > import org.apache.hadoop.conf.Configuration;
> >
> > import org.apache.hadoop.conf.Configured;
> >
> > import org.apache.hadoop.fs.Path;
> >
> > import org.apache.hadoop.io.BytesWritable;
> >
> > import org.apache.hadoop.io.IntWritable;
> >
> > import org.apache.hadoop.io.Text;
> >
> > import org.apache.hadoop.mapreduce.Job;
> >
> > import org.apache.hadoop.mapreduce.Mapper;
> >
> > import org.apache.hadoop.mapreduce.Reducer;
> >
> > import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> >
> > import org.apache.hadoop.util.Tool;
> >
> > import org.apache.hadoop.util.ToolRunner;
> >
> >
> >
> > import com.drew.imaging.ImageMetadataReader;
> >
> > import com.drew.imaging.ImageProcessingException;
> >
> > import com.drew.metadata.Directory;
> >
> > import com.drew.metadata.Metadata;
> >
> > import com.drew.metadata.exif.ExifIFD0Directory;
> >
> >
> >
> > public class ImageSummary extends Configured implements Tool {
> >
> >
> >
> > public static class ImageSourceMapper extends
> >
> > Mapper<Text, BytesWritable, Text,
> > IntWritable> {
> >
> >
> >
> > private static int tagId = 272;
> >
> > private static final IntWritable one = new
> > IntWritable(1);
> >
> >
> >
> > public void map(Text imageName, BytesWritable
> > imageBytes,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the map
> method,
> > image is "
> >
> > +
> > imageName.toString());
> >
> >
> >
> > byte[] imageInBytes =
> > imageBytes.getBytes();
> >
> > ByteArrayInputStream bais = new
> > ByteArrayInputStream(imageInBytes);
> >
> > BufferedInputStream bis = new
> > BufferedInputStream(bais);
> >
> >
> >
> > Metadata imageMD = null;
> >
> >
> >
> > try {
> >
> > imageMD =
> > ImageMetadataReader.readMetadata(bis, true);
> >
> > } catch (ImageProcessingException e)
> {
> >
> > // TODO Auto-generated
> catch
> > block
> >
> > System.out.println("Got
> an
> > ImageProcessingException !");
> >
> > e.printStackTrace();
> >
> > }
> >
> >
> >
> > Directory exifIFD0Directory = imageMD
> >
> >
> > .getDirectory(ExifIFD0Directory.class);
> >
> >
> >
> > String imageSource =
> > exifIFD0Directory.getString(tagId);
> >
> >
> >
> >
> System.out.println(imageName.toString()
> > + " is taken using "
> >
> > +
> imageSource);
> >
> >
> >
> > context.write(new Text(imageSource),
> > one);
> >
> >
> >
> > System.out.println("Returning from
> the
> > map method");
> >
> > }
> >
> > }
> >
> >
> >
> > public static class ImageSourceReducer extends
> >
> > Reducer<Text, IntWritable, Text,
> > IntWritable> {
> >
> >
> >
> > public void reduce(Text imageSource,
> > Iterator<IntWritable> counts,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the reduce
> > method");
> >
> >
> >
> > int finalCount = 0;
> >
> >
> >
> > while (counts.hasNext()) {
> >
> > finalCount +=
> > counts.next().get();
> >
> > }
> >
> >
> >
> > context.write(imageSource, new
> > IntWritable(finalCount));
> >
> >
> >
> > System.out.println("Returning from
> the
> > reduce method");
> >
> > }
> >
> >
> >
> > }
> >
> >
> >
> > public static void main(String[] args) throws Exception {
> >
> > ToolRunner.run(new ImageSummary(), args);
> >
> > }
> >
> >
> >
> > @Override
> >
> > public int run(String[] args) throws Exception {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In ImageSummary.run(...)");
> >
> >
> >
> > Configuration configuration = getConf();
> >
> >
> >
> > Job job = new Job(configuration, "Image_Source");
> >
> > job.setJarByClass(getClass());
> >
> >
> >
> >
> > job.setInputFormatClass(SequenceFileInputFormat.class);
> >
> > job.setOutputFormatClass(TextOutputFormat.class);
> >
> >
> >
> > job.setMapperClass(ImageSourceMapper.class);
> >
> > job.setCombinerClass(ImageSourceReducer.class);
> >
> > job.setReducerClass(ImageSourceReducer.class);
> >
> >
> >
> > job.setMapOutputKeyClass(Text.class);
> >
> > job.setMapOutputValueClass(IntWritable.class);
> >
> >
> >
> > job.setOutputKeyClass(Text.class);
> >
> > job.setOutputValueClass(IntWritable.class);
> >
> >
> >
> > SequenceFileInputFormat.addInputPath(job, new
> > Path(args[0]));
> >
> > TextOutputFormat.setOutputPath(job, new
> > Path(args[1]));
> >
> >
> >
> > System.out.println("Submitting job");
> >
> >
> >
> > job.waitForCompletion(true);
> >
> >
> >
> > int jobStatus = job.isSuccessful() ? 0 : -1;
> >
> >
> >
> > System.out.println("Returning jobStatus = " +
> > jobStatus);
> >
> >
> >
> > return jobStatus;
> >
> > }
> >
> > }
> >
> >
> >
> >
> >
> > The command :
> >
> >
> >
> > hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> > com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> > "/scratchpad/imageOps/cnt"
> >
> >
> >
> >
> >
> > The part-file (/scratchpad/imageOps/cnt/part-r-00000)
> >
> > COOLPIX L120 1
> >
> > COOLPIX L120 1
> >
> > K750i 1
> >
> >
> >
> > The mapper stdout logs :
> >
> >
> >
> > stdout logs
> >
> > In the map method, image is It's a long road....JPG
> >
> > It's a long road....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is Every man is a mountainside....JPG
> >
> > Every man is a mountainside....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is mystic.JPG
> >
> > mystic.JPG is taken using K750i
> >
> > Returning from the map method
> >
> > But nothing is reflected in stdout logs of the reducer.
> >
> > What have I missed?
> >
> >
> >
> > Regards,
> >
> > Omkar Joshi
> >
> >
> >
> >
> >
> >
> > ________________________________
> > The contents of this e-mail and any attachment(s) may contain
> confidential
> > or privileged information for the intended recipient(s). Unintended
> > recipients are prohibited from taking action on the basis of information
> in
> > this e-mail and using or disseminating the information, and must notify
> the
> > sender and delete it from their system. L&T Infotech will not accept
> > responsibility or liability for the accuracy or completeness of, or the
> > presence of any virus or disabling code in this e-mail"
>
>
>
> --
> Harsh J
>
Re: Reducer not getting called
Posted by Rahul Bhattacharjee <ra...@gmail.com>.
The programming error is already mentioned. You are actually not overriding
base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi
<Om...@lntinfotech.com>wrote:
> Ok but that link is broken - can you provide a working one?
>
> Regards,
> Omkar Joshi
>
>
> -----Original Message-----
> From: Harsh J [mailto:harsh@cloudera.com]
> Sent: Thursday, June 13, 2013 11:01 AM
> To: <us...@hadoop.apache.org>
> Subject: Re: Reducer not getting called
>
> You're not using the recommended @Override annotations, and are
> hitting a classic programming mistake. Your issue is same as this
> earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
> ones before it).
>
> On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
> <Om...@lntinfotech.com> wrote:
> > Hi,
> >
> >
> >
> > I have a SequenceFile which contains several jpeg images with (image
> name,
> > image bytes) as key-value pairs. My objective is to count the no. of
> images
> > by grouping them by the source, something like this :
> >
> >
> >
> > Nikon Coolpix 100
> >
> > Sony Cybershot 251
> >
> > N82 100
> >
> >
> >
> >
> >
> > The MR code is :
> >
> >
> >
> > package com.hadoop.basics;
> >
> >
> >
> > import java.io.BufferedInputStream;
> >
> > import java.io.ByteArrayInputStream;
> >
> > import java.io.IOException;
> >
> > import java.util.Iterator;
> >
> >
> >
> > import org.apache.hadoop.conf.Configuration;
> >
> > import org.apache.hadoop.conf.Configured;
> >
> > import org.apache.hadoop.fs.Path;
> >
> > import org.apache.hadoop.io.BytesWritable;
> >
> > import org.apache.hadoop.io.IntWritable;
> >
> > import org.apache.hadoop.io.Text;
> >
> > import org.apache.hadoop.mapreduce.Job;
> >
> > import org.apache.hadoop.mapreduce.Mapper;
> >
> > import org.apache.hadoop.mapreduce.Reducer;
> >
> > import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> >
> > import org.apache.hadoop.util.Tool;
> >
> > import org.apache.hadoop.util.ToolRunner;
> >
> >
> >
> > import com.drew.imaging.ImageMetadataReader;
> >
> > import com.drew.imaging.ImageProcessingException;
> >
> > import com.drew.metadata.Directory;
> >
> > import com.drew.metadata.Metadata;
> >
> > import com.drew.metadata.exif.ExifIFD0Directory;
> >
> >
> >
> > public class ImageSummary extends Configured implements Tool {
> >
> >
> >
> > public static class ImageSourceMapper extends
> >
> > Mapper<Text, BytesWritable, Text,
> > IntWritable> {
> >
> >
> >
> > private static int tagId = 272;
> >
> > private static final IntWritable one = new
> > IntWritable(1);
> >
> >
> >
> > public void map(Text imageName, BytesWritable
> > imageBytes,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the map
> method,
> > image is "
> >
> > +
> > imageName.toString());
> >
> >
> >
> > byte[] imageInBytes =
> > imageBytes.getBytes();
> >
> > ByteArrayInputStream bais = new
> > ByteArrayInputStream(imageInBytes);
> >
> > BufferedInputStream bis = new
> > BufferedInputStream(bais);
> >
> >
> >
> > Metadata imageMD = null;
> >
> >
> >
> > try {
> >
> > imageMD =
> > ImageMetadataReader.readMetadata(bis, true);
> >
> > } catch (ImageProcessingException e)
> {
> >
> > // TODO Auto-generated
> catch
> > block
> >
> > System.out.println("Got
> an
> > ImageProcessingException !");
> >
> > e.printStackTrace();
> >
> > }
> >
> >
> >
> > Directory exifIFD0Directory = imageMD
> >
> >
> > .getDirectory(ExifIFD0Directory.class);
> >
> >
> >
> > String imageSource =
> > exifIFD0Directory.getString(tagId);
> >
> >
> >
> >
> System.out.println(imageName.toString()
> > + " is taken using "
> >
> > +
> imageSource);
> >
> >
> >
> > context.write(new Text(imageSource),
> > one);
> >
> >
> >
> > System.out.println("Returning from
> the
> > map method");
> >
> > }
> >
> > }
> >
> >
> >
> > public static class ImageSourceReducer extends
> >
> > Reducer<Text, IntWritable, Text,
> > IntWritable> {
> >
> >
> >
> > public void reduce(Text imageSource,
> > Iterator<IntWritable> counts,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the reduce
> > method");
> >
> >
> >
> > int finalCount = 0;
> >
> >
> >
> > while (counts.hasNext()) {
> >
> > finalCount +=
> > counts.next().get();
> >
> > }
> >
> >
> >
> > context.write(imageSource, new
> > IntWritable(finalCount));
> >
> >
> >
> > System.out.println("Returning from
> the
> > reduce method");
> >
> > }
> >
> >
> >
> > }
> >
> >
> >
> > public static void main(String[] args) throws Exception {
> >
> > ToolRunner.run(new ImageSummary(), args);
> >
> > }
> >
> >
> >
> > @Override
> >
> > public int run(String[] args) throws Exception {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In ImageSummary.run(...)");
> >
> >
> >
> > Configuration configuration = getConf();
> >
> >
> >
> > Job job = new Job(configuration, "Image_Source");
> >
> > job.setJarByClass(getClass());
> >
> >
> >
> >
> > job.setInputFormatClass(SequenceFileInputFormat.class);
> >
> > job.setOutputFormatClass(TextOutputFormat.class);
> >
> >
> >
> > job.setMapperClass(ImageSourceMapper.class);
> >
> > job.setCombinerClass(ImageSourceReducer.class);
> >
> > job.setReducerClass(ImageSourceReducer.class);
> >
> >
> >
> > job.setMapOutputKeyClass(Text.class);
> >
> > job.setMapOutputValueClass(IntWritable.class);
> >
> >
> >
> > job.setOutputKeyClass(Text.class);
> >
> > job.setOutputValueClass(IntWritable.class);
> >
> >
> >
> > SequenceFileInputFormat.addInputPath(job, new
> > Path(args[0]));
> >
> > TextOutputFormat.setOutputPath(job, new
> > Path(args[1]));
> >
> >
> >
> > System.out.println("Submitting job");
> >
> >
> >
> > job.waitForCompletion(true);
> >
> >
> >
> > int jobStatus = job.isSuccessful() ? 0 : -1;
> >
> >
> >
> > System.out.println("Returning jobStatus = " +
> > jobStatus);
> >
> >
> >
> > return jobStatus;
> >
> > }
> >
> > }
> >
> >
> >
> >
> >
> > The command :
> >
> >
> >
> > hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> > com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> > "/scratchpad/imageOps/cnt"
> >
> >
> >
> >
> >
> > The part-file (/scratchpad/imageOps/cnt/part-r-00000)
> >
> > COOLPIX L120 1
> >
> > COOLPIX L120 1
> >
> > K750i 1
> >
> >
> >
> > The mapper stdout logs :
> >
> >
> >
> > stdout logs
> >
> > In the map method, image is It's a long road....JPG
> >
> > It's a long road....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is Every man is a mountainside....JPG
> >
> > Every man is a mountainside....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is mystic.JPG
> >
> > mystic.JPG is taken using K750i
> >
> > Returning from the map method
> >
> > But nothing is reflected in stdout logs of the reducer.
> >
> > What have I missed?
> >
> >
> >
> > Regards,
> >
> > Omkar Joshi
> >
> >
> >
> >
> >
> >
> > ________________________________
> > The contents of this e-mail and any attachment(s) may contain
> confidential
> > or privileged information for the intended recipient(s). Unintended
> > recipients are prohibited from taking action on the basis of information
> in
> > this e-mail and using or disseminating the information, and must notify
> the
> > sender and delete it from their system. L&T Infotech will not accept
> > responsibility or liability for the accuracy or completeness of, or the
> > presence of any virus or disabling code in this e-mail"
>
>
>
> --
> Harsh J
>
Re: Reducer not getting called
Posted by Rahul Bhattacharjee <ra...@gmail.com>.
The programming error is already mentioned. You are actually not overriding
base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi
<Om...@lntinfotech.com>wrote:
> Ok but that link is broken - can you provide a working one?
>
> Regards,
> Omkar Joshi
>
>
> -----Original Message-----
> From: Harsh J [mailto:harsh@cloudera.com]
> Sent: Thursday, June 13, 2013 11:01 AM
> To: <us...@hadoop.apache.org>
> Subject: Re: Reducer not getting called
>
> You're not using the recommended @Override annotations, and are
> hitting a classic programming mistake. Your issue is same as this
> earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
> ones before it).
>
> On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
> <Om...@lntinfotech.com> wrote:
> > Hi,
> >
> >
> >
> > I have a SequenceFile which contains several jpeg images with (image
> name,
> > image bytes) as key-value pairs. My objective is to count the no. of
> images
> > by grouping them by the source, something like this :
> >
> >
> >
> > Nikon Coolpix 100
> >
> > Sony Cybershot 251
> >
> > N82 100
> >
> >
> >
> >
> >
> > The MR code is :
> >
> >
> >
> > package com.hadoop.basics;
> >
> >
> >
> > import java.io.BufferedInputStream;
> >
> > import java.io.ByteArrayInputStream;
> >
> > import java.io.IOException;
> >
> > import java.util.Iterator;
> >
> >
> >
> > import org.apache.hadoop.conf.Configuration;
> >
> > import org.apache.hadoop.conf.Configured;
> >
> > import org.apache.hadoop.fs.Path;
> >
> > import org.apache.hadoop.io.BytesWritable;
> >
> > import org.apache.hadoop.io.IntWritable;
> >
> > import org.apache.hadoop.io.Text;
> >
> > import org.apache.hadoop.mapreduce.Job;
> >
> > import org.apache.hadoop.mapreduce.Mapper;
> >
> > import org.apache.hadoop.mapreduce.Reducer;
> >
> > import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> >
> > import org.apache.hadoop.util.Tool;
> >
> > import org.apache.hadoop.util.ToolRunner;
> >
> >
> >
> > import com.drew.imaging.ImageMetadataReader;
> >
> > import com.drew.imaging.ImageProcessingException;
> >
> > import com.drew.metadata.Directory;
> >
> > import com.drew.metadata.Metadata;
> >
> > import com.drew.metadata.exif.ExifIFD0Directory;
> >
> >
> >
> > public class ImageSummary extends Configured implements Tool {
> >
> >
> >
> > public static class ImageSourceMapper extends
> >
> > Mapper<Text, BytesWritable, Text,
> > IntWritable> {
> >
> >
> >
> > private static int tagId = 272;
> >
> > private static final IntWritable one = new
> > IntWritable(1);
> >
> >
> >
> > public void map(Text imageName, BytesWritable
> > imageBytes,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the map
> method,
> > image is "
> >
> > +
> > imageName.toString());
> >
> >
> >
> > byte[] imageInBytes =
> > imageBytes.getBytes();
> >
> > ByteArrayInputStream bais = new
> > ByteArrayInputStream(imageInBytes);
> >
> > BufferedInputStream bis = new
> > BufferedInputStream(bais);
> >
> >
> >
> > Metadata imageMD = null;
> >
> >
> >
> > try {
> >
> > imageMD =
> > ImageMetadataReader.readMetadata(bis, true);
> >
> > } catch (ImageProcessingException e)
> {
> >
> > // TODO Auto-generated
> catch
> > block
> >
> > System.out.println("Got
> an
> > ImageProcessingException !");
> >
> > e.printStackTrace();
> >
> > }
> >
> >
> >
> > Directory exifIFD0Directory = imageMD
> >
> >
> > .getDirectory(ExifIFD0Directory.class);
> >
> >
> >
> > String imageSource =
> > exifIFD0Directory.getString(tagId);
> >
> >
> >
> >
> System.out.println(imageName.toString()
> > + " is taken using "
> >
> > +
> imageSource);
> >
> >
> >
> > context.write(new Text(imageSource),
> > one);
> >
> >
> >
> > System.out.println("Returning from
> the
> > map method");
> >
> > }
> >
> > }
> >
> >
> >
> > public static class ImageSourceReducer extends
> >
> > Reducer<Text, IntWritable, Text,
> > IntWritable> {
> >
> >
> >
> > public void reduce(Text imageSource,
> > Iterator<IntWritable> counts,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the reduce
> > method");
> >
> >
> >
> > int finalCount = 0;
> >
> >
> >
> > while (counts.hasNext()) {
> >
> > finalCount +=
> > counts.next().get();
> >
> > }
> >
> >
> >
> > context.write(imageSource, new
> > IntWritable(finalCount));
> >
> >
> >
> > System.out.println("Returning from
> the
> > reduce method");
> >
> > }
> >
> >
> >
> > }
> >
> >
> >
> > public static void main(String[] args) throws Exception {
> >
> > ToolRunner.run(new ImageSummary(), args);
> >
> > }
> >
> >
> >
> > @Override
> >
> > public int run(String[] args) throws Exception {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In ImageSummary.run(...)");
> >
> >
> >
> > Configuration configuration = getConf();
> >
> >
> >
> > Job job = new Job(configuration, "Image_Source");
> >
> > job.setJarByClass(getClass());
> >
> >
> >
> >
> > job.setInputFormatClass(SequenceFileInputFormat.class);
> >
> > job.setOutputFormatClass(TextOutputFormat.class);
> >
> >
> >
> > job.setMapperClass(ImageSourceMapper.class);
> >
> > job.setCombinerClass(ImageSourceReducer.class);
> >
> > job.setReducerClass(ImageSourceReducer.class);
> >
> >
> >
> > job.setMapOutputKeyClass(Text.class);
> >
> > job.setMapOutputValueClass(IntWritable.class);
> >
> >
> >
> > job.setOutputKeyClass(Text.class);
> >
> > job.setOutputValueClass(IntWritable.class);
> >
> >
> >
> > SequenceFileInputFormat.addInputPath(job, new
> > Path(args[0]));
> >
> > TextOutputFormat.setOutputPath(job, new
> > Path(args[1]));
> >
> >
> >
> > System.out.println("Submitting job");
> >
> >
> >
> > job.waitForCompletion(true);
> >
> >
> >
> > int jobStatus = job.isSuccessful() ? 0 : -1;
> >
> >
> >
> > System.out.println("Returning jobStatus = " +
> > jobStatus);
> >
> >
> >
> > return jobStatus;
> >
> > }
> >
> > }
> >
> >
> >
> >
> >
> > The command :
> >
> >
> >
> > hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> > com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> > "/scratchpad/imageOps/cnt"
> >
> >
> >
> >
> >
> > The part-file (/scratchpad/imageOps/cnt/part-r-00000)
> >
> > COOLPIX L120 1
> >
> > COOLPIX L120 1
> >
> > K750i 1
> >
> >
> >
> > The mapper stdout logs :
> >
> >
> >
> > stdout logs
> >
> > In the map method, image is It's a long road....JPG
> >
> > It's a long road....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is Every man is a mountainside....JPG
> >
> > Every man is a mountainside....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is mystic.JPG
> >
> > mystic.JPG is taken using K750i
> >
> > Returning from the map method
> >
> > But nothing is reflected in stdout logs of the reducer.
> >
> > What have I missed?
> >
> >
> >
> > Regards,
> >
> > Omkar Joshi
> >
> >
> >
> >
> >
> >
> > ________________________________
> > The contents of this e-mail and any attachment(s) may contain
> confidential
> > or privileged information for the intended recipient(s). Unintended
> > recipients are prohibited from taking action on the basis of information
> in
> > this e-mail and using or disseminating the information, and must notify
> the
> > sender and delete it from their system. L&T Infotech will not accept
> > responsibility or liability for the accuracy or completeness of, or the
> > presence of any virus or disabling code in this e-mail"
>
>
>
> --
> Harsh J
>
Re: Reducer not getting called
Posted by Rahul Bhattacharjee <ra...@gmail.com>.
The programming error is already mentioned. You are actually not overriding
base classes method , rather created a new method.
Thanks,
Rahul
On Thu, Jun 13, 2013 at 11:12 AM, Omkar Joshi
<Om...@lntinfotech.com>wrote:
> Ok but that link is broken - can you provide a working one?
>
> Regards,
> Omkar Joshi
>
>
> -----Original Message-----
> From: Harsh J [mailto:harsh@cloudera.com]
> Sent: Thursday, June 13, 2013 11:01 AM
> To: <us...@hadoop.apache.org>
> Subject: Re: Reducer not getting called
>
> You're not using the recommended @Override annotations, and are
> hitting a classic programming mistake. Your issue is same as this
> earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
> ones before it).
>
> On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
> <Om...@lntinfotech.com> wrote:
> > Hi,
> >
> >
> >
> > I have a SequenceFile which contains several jpeg images with (image
> name,
> > image bytes) as key-value pairs. My objective is to count the no. of
> images
> > by grouping them by the source, something like this :
> >
> >
> >
> > Nikon Coolpix 100
> >
> > Sony Cybershot 251
> >
> > N82 100
> >
> >
> >
> >
> >
> > The MR code is :
> >
> >
> >
> > package com.hadoop.basics;
> >
> >
> >
> > import java.io.BufferedInputStream;
> >
> > import java.io.ByteArrayInputStream;
> >
> > import java.io.IOException;
> >
> > import java.util.Iterator;
> >
> >
> >
> > import org.apache.hadoop.conf.Configuration;
> >
> > import org.apache.hadoop.conf.Configured;
> >
> > import org.apache.hadoop.fs.Path;
> >
> > import org.apache.hadoop.io.BytesWritable;
> >
> > import org.apache.hadoop.io.IntWritable;
> >
> > import org.apache.hadoop.io.Text;
> >
> > import org.apache.hadoop.mapreduce.Job;
> >
> > import org.apache.hadoop.mapreduce.Mapper;
> >
> > import org.apache.hadoop.mapreduce.Reducer;
> >
> > import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> >
> > import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> >
> > import org.apache.hadoop.util.Tool;
> >
> > import org.apache.hadoop.util.ToolRunner;
> >
> >
> >
> > import com.drew.imaging.ImageMetadataReader;
> >
> > import com.drew.imaging.ImageProcessingException;
> >
> > import com.drew.metadata.Directory;
> >
> > import com.drew.metadata.Metadata;
> >
> > import com.drew.metadata.exif.ExifIFD0Directory;
> >
> >
> >
> > public class ImageSummary extends Configured implements Tool {
> >
> >
> >
> > public static class ImageSourceMapper extends
> >
> > Mapper<Text, BytesWritable, Text,
> > IntWritable> {
> >
> >
> >
> > private static int tagId = 272;
> >
> > private static final IntWritable one = new
> > IntWritable(1);
> >
> >
> >
> > public void map(Text imageName, BytesWritable
> > imageBytes,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the map
> method,
> > image is "
> >
> > +
> > imageName.toString());
> >
> >
> >
> > byte[] imageInBytes =
> > imageBytes.getBytes();
> >
> > ByteArrayInputStream bais = new
> > ByteArrayInputStream(imageInBytes);
> >
> > BufferedInputStream bis = new
> > BufferedInputStream(bais);
> >
> >
> >
> > Metadata imageMD = null;
> >
> >
> >
> > try {
> >
> > imageMD =
> > ImageMetadataReader.readMetadata(bis, true);
> >
> > } catch (ImageProcessingException e)
> {
> >
> > // TODO Auto-generated
> catch
> > block
> >
> > System.out.println("Got
> an
> > ImageProcessingException !");
> >
> > e.printStackTrace();
> >
> > }
> >
> >
> >
> > Directory exifIFD0Directory = imageMD
> >
> >
> > .getDirectory(ExifIFD0Directory.class);
> >
> >
> >
> > String imageSource =
> > exifIFD0Directory.getString(tagId);
> >
> >
> >
> >
> System.out.println(imageName.toString()
> > + " is taken using "
> >
> > +
> imageSource);
> >
> >
> >
> > context.write(new Text(imageSource),
> > one);
> >
> >
> >
> > System.out.println("Returning from
> the
> > map method");
> >
> > }
> >
> > }
> >
> >
> >
> > public static class ImageSourceReducer extends
> >
> > Reducer<Text, IntWritable, Text,
> > IntWritable> {
> >
> >
> >
> > public void reduce(Text imageSource,
> > Iterator<IntWritable> counts,
> >
> > Context context) throws
> > IOException, InterruptedException {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In the reduce
> > method");
> >
> >
> >
> > int finalCount = 0;
> >
> >
> >
> > while (counts.hasNext()) {
> >
> > finalCount +=
> > counts.next().get();
> >
> > }
> >
> >
> >
> > context.write(imageSource, new
> > IntWritable(finalCount));
> >
> >
> >
> > System.out.println("Returning from
> the
> > reduce method");
> >
> > }
> >
> >
> >
> > }
> >
> >
> >
> > public static void main(String[] args) throws Exception {
> >
> > ToolRunner.run(new ImageSummary(), args);
> >
> > }
> >
> >
> >
> > @Override
> >
> > public int run(String[] args) throws Exception {
> >
> > // TODO Auto-generated method stub
> >
> >
> >
> > System.out.println("In ImageSummary.run(...)");
> >
> >
> >
> > Configuration configuration = getConf();
> >
> >
> >
> > Job job = new Job(configuration, "Image_Source");
> >
> > job.setJarByClass(getClass());
> >
> >
> >
> >
> > job.setInputFormatClass(SequenceFileInputFormat.class);
> >
> > job.setOutputFormatClass(TextOutputFormat.class);
> >
> >
> >
> > job.setMapperClass(ImageSourceMapper.class);
> >
> > job.setCombinerClass(ImageSourceReducer.class);
> >
> > job.setReducerClass(ImageSourceReducer.class);
> >
> >
> >
> > job.setMapOutputKeyClass(Text.class);
> >
> > job.setMapOutputValueClass(IntWritable.class);
> >
> >
> >
> > job.setOutputKeyClass(Text.class);
> >
> > job.setOutputValueClass(IntWritable.class);
> >
> >
> >
> > SequenceFileInputFormat.addInputPath(job, new
> > Path(args[0]));
> >
> > TextOutputFormat.setOutputPath(job, new
> > Path(args[1]));
> >
> >
> >
> > System.out.println("Submitting job");
> >
> >
> >
> > job.waitForCompletion(true);
> >
> >
> >
> > int jobStatus = job.isSuccessful() ? 0 : -1;
> >
> >
> >
> > System.out.println("Returning jobStatus = " +
> > jobStatus);
> >
> >
> >
> > return jobStatus;
> >
> > }
> >
> > }
> >
> >
> >
> >
> >
> > The command :
> >
> >
> >
> > hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> > com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> > "/scratchpad/imageOps/cnt"
> >
> >
> >
> >
> >
> > The part-file (/scratchpad/imageOps/cnt/part-r-00000)
> >
> > COOLPIX L120 1
> >
> > COOLPIX L120 1
> >
> > K750i 1
> >
> >
> >
> > The mapper stdout logs :
> >
> >
> >
> > stdout logs
> >
> > In the map method, image is It's a long road....JPG
> >
> > It's a long road....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is Every man is a mountainside....JPG
> >
> > Every man is a mountainside....JPG is taken using COOLPIX L120
> >
> > Returning from the map method
> >
> > In the map method, image is mystic.JPG
> >
> > mystic.JPG is taken using K750i
> >
> > Returning from the map method
> >
> > But nothing is reflected in stdout logs of the reducer.
> >
> > What have I missed?
> >
> >
> >
> > Regards,
> >
> > Omkar Joshi
> >
> >
> >
> >
> >
> >
> > ________________________________
> > The contents of this e-mail and any attachment(s) may contain
> confidential
> > or privileged information for the intended recipient(s). Unintended
> > recipients are prohibited from taking action on the basis of information
> in
> > this e-mail and using or disseminating the information, and must notify
> the
> > sender and delete it from their system. L&T Infotech will not accept
> > responsibility or liability for the accuracy or completeness of, or the
> > presence of any virus or disabling code in this e-mail"
>
>
>
> --
> Harsh J
>
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
RE: Reducer not getting called
Posted by Omkar Joshi <Om...@lntinfotech.com>.
Ok but that link is broken - can you provide a working one?
Regards,
Omkar Joshi
-----Original Message-----
From: Harsh J [mailto:harsh@cloudera.com]
Sent: Thursday, June 13, 2013 11:01 AM
To: <us...@hadoop.apache.org>
Subject: Re: Reducer not getting called
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
Re: Reducer not getting called
Posted by Harsh J <ha...@cloudera.com>.
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
Re: Reducer not getting called
Posted by Harsh J <ha...@cloudera.com>.
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
Re: Reducer not getting called
Posted by Harsh J <ha...@cloudera.com>.
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J
Re: Reducer not getting called
Posted by Harsh J <ha...@cloudera.com>.
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).
On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Om...@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix 100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
> public static class ImageSourceMapper extends
>
> Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
> private static int tagId = 272;
>
> private static final IntWritable one = new
> IntWritable(1);
>
>
>
> public void map(Text imageName, BytesWritable
> imageBytes,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the map method,
> image is "
>
> +
> imageName.toString());
>
>
>
> byte[] imageInBytes =
> imageBytes.getBytes();
>
> ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
> BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
> Metadata imageMD = null;
>
>
>
> try {
>
> imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
> } catch (ImageProcessingException e) {
>
> // TODO Auto-generated catch
> block
>
> System.out.println("Got an
> ImageProcessingException !");
>
> e.printStackTrace();
>
> }
>
>
>
> Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
> String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
> System.out.println(imageName.toString()
> + " is taken using "
>
> + imageSource);
>
>
>
> context.write(new Text(imageSource),
> one);
>
>
>
> System.out.println("Returning from the
> map method");
>
> }
>
> }
>
>
>
> public static class ImageSourceReducer extends
>
> Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
> public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
> Context context) throws
> IOException, InterruptedException {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In the reduce
> method");
>
>
>
> int finalCount = 0;
>
>
>
> while (counts.hasNext()) {
>
> finalCount +=
> counts.next().get();
>
> }
>
>
>
> context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
> System.out.println("Returning from the
> reduce method");
>
> }
>
>
>
> }
>
>
>
> public static void main(String[] args) throws Exception {
>
> ToolRunner.run(new ImageSummary(), args);
>
> }
>
>
>
> @Override
>
> public int run(String[] args) throws Exception {
>
> // TODO Auto-generated method stub
>
>
>
> System.out.println("In ImageSummary.run(...)");
>
>
>
> Configuration configuration = getConf();
>
>
>
> Job job = new Job(configuration, "Image_Source");
>
> job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
> job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
> job.setMapperClass(ImageSourceMapper.class);
>
> job.setCombinerClass(ImageSourceReducer.class);
>
> job.setReducerClass(ImageSourceReducer.class);
>
>
>
> job.setMapOutputKeyClass(Text.class);
>
> job.setMapOutputValueClass(IntWritable.class);
>
>
>
> job.setOutputKeyClass(Text.class);
>
> job.setOutputValueClass(IntWritable.class);
>
>
>
> SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
> TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
> System.out.println("Submitting job");
>
>
>
> job.waitForCompletion(true);
>
>
>
> int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
> System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
> return jobStatus;
>
> }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120 1
>
> COOLPIX L120 1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"
--
Harsh J