You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hbase.apache.org by steven zhuang <zh...@gmail.com> on 2010/02/25 03:46:06 UTC
can't load configuration successfully with the newest mapreduce
package.
hello,
I have a program loading formated data into HTable, but it crashed
when connecting to the quorum server because of wrong configuration, seems
that it load configuration from hbase-default.xml, not from hbase-site.xml.
my hadoop cluster has 35 nodes, and the administrator has set up 4
quorum server and 5 region server(too few?). we have copied the Hbase
configuration folder to every hadoop task tracker.
the job failed during the reduce phase, raised exceptions "
NoServerForRegionException", I have checked the job log on some nodes, the
connection string is set to "localhost:port" instead of the configuration in
hbase-site.xml.
my code is as follows, I think the problem may be from the *doJob
method(when I do the configuration):*
please help.
*
*
*import java.io.IOException;*
*import java.util.Iterator;*
*
*
*import org.apache.hadoop.conf.Configuration;*
*import org.apache.hadoop.conf.Configured;*
*import org.apache.hadoop.hbase.HBaseConfiguration;*
*import org.apache.hadoop.hbase.HColumnDescriptor;*
*import org.apache.hadoop.hbase.HTableDescriptor;*
*import org.apache.hadoop.hbase.TableExistsException;*
*import org.apache.hadoop.hbase.client.HBaseAdmin;*
*import org.apache.hadoop.hbase.client.Put;*
*import org.apache.hadoop.hbase.io.HbaseMapWritable;*
*import org.apache.hadoop.hbase.io.ImmutableBytesWritable;*
*import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;*
*import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;*
*import org.apache.hadoop.hbase.mapreduce.TableReducer;*
*import org.apache.hadoop.hbase.util.Bytes;*
*import org.apache.hadoop.io.LongWritable;*
*import org.apache.hadoop.io.Text;*
*import org.apache.hadoop.mapred.JobClient;*
*import org.apache.hadoop.mapred.JobConf;*
*import org.apache.hadoop.mapreduce.Job;*
*import org.apache.hadoop.mapreduce.Mapper;*
*import org.apache.hadoop.mapreduce.Mapper.Context;*
*import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;*
*import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;*
*import org.apache.hadoop.util.Tool;*
*
*
*public class FasterMidImport extends Configured {*
*
*
*......*
* **private static Configuration conf = new HBaseConfiguration();*
*
*
* **public static class InnerMap extends Mapper<LongWritable, Text, Text,
Text> {*
*
*
* **protected void map(LongWritable key, Text value, Context context)*
* **throws IOException, InterruptedException {*
* ......*
* **context.write(new Text(rowKey), new Text(columnid + "\t" + value));*
* **}*
* **}*
*
*
* **public static class MyTableReducer extends TableReducer<Text, Text,
Text> {*
* **protected void reduce(Text key, Iterable<Text> values, Context context)*
* **throws IOException, InterruptedException {*
* **Iterator<Text> iter = values.iterator();*
* **Put put = new Put(key.toString().getBytes());*
* **int cellCount = 0;*
* **while (iter.hasNext()) {*
* **String[] splits = iter.next().toString().split("\t");*
* **if (splits.length < 2) {*
* **continue;*
* **}*
* **String query = splits[0].substring(splits[0].indexOf(':') + 1);*
* **put.add("queries".getBytes(), query.getBytes(), splits[1]*
* **.getBytes());*
* **if (++cellCount > 10000) {*
* **cellCount = 0;*
* **context.write(key, put);*
* **put = new Put(key.toString().getBytes());*
* **}*
* **}*
* **if (0 < cellCount) {*
* **context.write(key, put);*
* **}*
* **}*
* **}*
*
*
* **public void doJob(String[] args) throws IOException {*
* **conf.set("input.dir", args[0]);*
* **conf.set("table.name", args[1]);*
* **try {*
* **String tblName = conf.get("table.name");*
* **try {*
* **HBaseAdmin admin = new HBaseAdmin((HBaseConfiguration) conf);*
* **if (!admin.tableExists(tblName)) {*
* **Thread.currentThread().sleep(2000);*
* **HTableDescriptor tblDesc = new HTableDescriptor(tblName);*
* **tblDesc.addFamily(new HColumnDescriptor("queries"));*
* **admin.createTable(tblDesc);*
* **}*
* **} catch (TableExistsException e) {*
* **} catch (InterruptedException e) {*
* **e.printStackTrace();*
* **}*
* **} catch (IOException e) {*
* **e.printStackTrace();*
* **}*
* **Job job = new Job(conf);*
* **// Job job = new Job(new Configuration());*
* **job.setJarByClass(FasterMidImport.class);*
* **job.setJobName("Faster-midImport: import " + conf.get("input.dir")*
* **+ "into" + conf.get("input.dir"));*
*
*
* **job.setOutputKeyClass(Text.class);*
* **job.setOutputValueClass(Text.class);*
* **job.setOutputFormatClass(TableOutputFormat.class);*
*
*
* **job.setInputFormatClass(TextInputFormat.class);*
* **FileInputFormat.setInputPaths(job, conf.get("input.dir"));*
*
*
* **job.setMapperClass(InnerMap.class);*
*
*
* **job.setMapOutputKeyClass(Text.class);*
* **job.setMapOutputValueClass(Text.class);*
*
*
* **job.setNumReduceTasks(20);*
* **//
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
*
* **TableMapReduceUtil.initTableReducerJob(conf.get("table.name"),*
* **MyTableReducer.class, job);*
* **try {*
* **job.submit();*
* **while (!job.isComplete()) {*
* **Thread.currentThread().sleep(1000);*
* **System.out*
* **.println("Map: " + (job.mapProgress() * 100)*
* **+ "% ... Reduce: "*
* **+ (job.reduceProgress() * 100) + "%");*
*
*
* **}*
* **} catch (InterruptedException e) {*
* **e.printStackTrace();*
* **} catch (ClassNotFoundException e) {*
* **e.printStackTrace();*
* **}*
* **}*
* **static int printUsage() {*
* **System.err.println("Usage: FasterMidImport <input-dir> <table_name>");*
* **return -1;*
* **}*
* **/***
* ** * * Run*
* ** * */*
* **public int run(String[] args) throws Exception {*
* **if (args.length < 2) {*
* **return printUsage();*
* **}*
* **this.doJob(args);*
* **return 0;*
* **}*
*
*
* **public static int main(String[] args) throws Exception {*
* **new FasterMidImport().run(args);*
* **return 0;*
* **}*
*}*
--
best wishes.
steven
Re: can't load configuration successfully with the newest mapreduce
package.
Posted by steven zhuang <zh...@gmail.com>.
It is not me who did the HADOOP configuration, the HADOOP_CLASSPATH is
set to:
HADOOP_CLASSPATH=$HBASE_HOME/hbase-0.20.3.jar:$HBASE_HOME/hbase-0.20.3-test.jar:$HBASE_HOME/lib/zookeeper-3.2.2.jar:$HBASE_HOME/conf
Instead of using the build hbase jar, it uses the jar files from
distribution, any problem with this?
The working code uses the Tool+ToolRunner+JobConf classes from the
mapred package, it correctly loaded the configuration.
On Thu, Feb 25, 2010 at 11:08 AM, steven zhuang <zh...@gmail.com>wrote:
> yeah, the region server is easily overloaded at present, but is it
> necessary to put a region server with each data node?
>
> and can anybody tell me the difference between the following two
> statements?
> *A: **private static Configuration conf = new HBaseConfiguration();*
> * **Job job = new Job(conf);*
> *##################################################################*
> *B: **// Job job = new Job(new Configuration());*
>
>
> I am still tracking why my code didn't work out.
>
>
> --
> best wishes.
> steven
>
>
--
best wishes.
steven
Re: can't load configuration successfully with the newest mapreduce
package.
Posted by steven zhuang <zh...@gmail.com>.
thank you, J.D.
On Thu, Feb 25, 2010 at 12:14 PM, Jean-Daniel Cryans <jd...@apache.org>wrote:
> More region servers means the data is more distributed, each really
> serves its share of the total load.
>
> Statement A will load all HBase-related configurations, B will not.
>
> On Wed, Feb 24, 2010 at 7:08 PM, steven zhuang <zh...@gmail.com>
> wrote:
> > yeah, the region server is easily overloaded at present, but is it
> necessary
> > to put a region server with each data node?
> >
> > and can anybody tell me the difference between the following two
> > statements?
> > *A: **private static Configuration conf = new HBaseConfiguration();*
> > * **Job job = new Job(conf);*
> > *##################################################################*
> > *B: **// Job job = new Job(new Configuration());*
> >
> >
> > I am still tracking why my code didn't work out.
> >
> >
> > --
> > best wishes.
> > steven
> >
>
--
best wishes.
steven
Re: can't load configuration successfully with the newest mapreduce
package.
Posted by Jean-Daniel Cryans <jd...@apache.org>.
More region servers means the data is more distributed, each really
serves its share of the total load.
Statement A will load all HBase-related configurations, B will not.
On Wed, Feb 24, 2010 at 7:08 PM, steven zhuang <zh...@gmail.com> wrote:
> yeah, the region server is easily overloaded at present, but is it necessary
> to put a region server with each data node?
>
> and can anybody tell me the difference between the following two
> statements?
> *A: **private static Configuration conf = new HBaseConfiguration();*
> * **Job job = new Job(conf);*
> *##################################################################*
> *B: **// Job job = new Job(new Configuration());*
>
>
> I am still tracking why my code didn't work out.
>
>
> --
> best wishes.
> steven
>
Re: can't load configuration successfully with the newest mapreduce
package.
Posted by steven zhuang <zh...@gmail.com>.
yeah, the region server is easily overloaded at present, but is it necessary
to put a region server with each data node?
and can anybody tell me the difference between the following two
statements?
*A: **private static Configuration conf = new HBaseConfiguration();*
* **Job job = new Job(conf);*
*##################################################################*
*B: **// Job job = new Job(new Configuration());*
I am still tracking why my code didn't work out.
--
best wishes.
steven
Re: can't load configuration successfully with the newest mapreduce
package.
Posted by Jean-Daniel Cryans <jd...@apache.org>.
Steven,
I would recommend putting 1 region server with each datanode and use 3
or 5 quorum servers, 4 means that your quorum is already crippled of
one member.
I would also recommend reviewing the following doc:
http://hadoop.apache.org/hbase/docs/r0.20.3/api/org/apache/hadoop/hbase/mapreduce/package-summary.html
See how the classpath is handled and try to do the same. It's how I
personally do it.
J-D
On Wed, Feb 24, 2010 at 6:46 PM, steven zhuang <zh...@gmail.com> wrote:
> hello,
> I have a program loading formated data into HTable, but it crashed
> when connecting to the quorum server because of wrong configuration, seems
> that it load configuration from hbase-default.xml, not from hbase-site.xml.
>
> my hadoop cluster has 35 nodes, and the administrator has set up 4
> quorum server and 5 region server(too few?). we have copied the Hbase
> configuration folder to every hadoop task tracker.
>
> the job failed during the reduce phase, raised exceptions "
> NoServerForRegionException", I have checked the job log on some nodes, the
> connection string is set to "localhost:port" instead of the configuration in
> hbase-site.xml.
>
> my code is as follows, I think the problem may be from the *doJob
> method(when I do the configuration):*
> please help.
> *
> *
> *import java.io.IOException;*
> *import java.util.Iterator;*
> *
> *
> *import org.apache.hadoop.conf.Configuration;*
> *import org.apache.hadoop.conf.Configured;*
> *import org.apache.hadoop.hbase.HBaseConfiguration;*
> *import org.apache.hadoop.hbase.HColumnDescriptor;*
> *import org.apache.hadoop.hbase.HTableDescriptor;*
> *import org.apache.hadoop.hbase.TableExistsException;*
> *import org.apache.hadoop.hbase.client.HBaseAdmin;*
> *import org.apache.hadoop.hbase.client.Put;*
> *import org.apache.hadoop.hbase.io.HbaseMapWritable;*
> *import org.apache.hadoop.hbase.io.ImmutableBytesWritable;*
> *import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;*
> *import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;*
> *import org.apache.hadoop.hbase.mapreduce.TableReducer;*
> *import org.apache.hadoop.hbase.util.Bytes;*
> *import org.apache.hadoop.io.LongWritable;*
> *import org.apache.hadoop.io.Text;*
> *import org.apache.hadoop.mapred.JobClient;*
> *import org.apache.hadoop.mapred.JobConf;*
> *import org.apache.hadoop.mapreduce.Job;*
> *import org.apache.hadoop.mapreduce.Mapper;*
> *import org.apache.hadoop.mapreduce.Mapper.Context;*
> *import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;*
> *import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;*
> *import org.apache.hadoop.util.Tool;*
> *
> *
> *public class FasterMidImport extends Configured {*
> *
> *
> *......*
> * **private static Configuration conf = new HBaseConfiguration();*
> *
> *
> * **public static class InnerMap extends Mapper<LongWritable, Text, Text,
> Text> {*
> *
> *
> * **protected void map(LongWritable key, Text value, Context context)*
> * **throws IOException, InterruptedException {*
> * ......*
> * **context.write(new Text(rowKey), new Text(columnid + "\t" + value));*
> * **}*
> * **}*
> *
> *
> * **public static class MyTableReducer extends TableReducer<Text, Text,
> Text> {*
> * **protected void reduce(Text key, Iterable<Text> values, Context context)*
> * **throws IOException, InterruptedException {*
> * **Iterator<Text> iter = values.iterator();*
> * **Put put = new Put(key.toString().getBytes());*
> * **int cellCount = 0;*
> * **while (iter.hasNext()) {*
> * **String[] splits = iter.next().toString().split("\t");*
> * **if (splits.length < 2) {*
> * **continue;*
> * **}*
> * **String query = splits[0].substring(splits[0].indexOf(':') + 1);*
> * **put.add("queries".getBytes(), query.getBytes(), splits[1]*
> * **.getBytes());*
> * **if (++cellCount > 10000) {*
> * **cellCount = 0;*
> * **context.write(key, put);*
> * **put = new Put(key.toString().getBytes());*
> * **}*
> * **}*
> * **if (0 < cellCount) {*
> * **context.write(key, put);*
> * **}*
> * **}*
> * **}*
> *
> *
> * **public void doJob(String[] args) throws IOException {*
> * **conf.set("input.dir", args[0]);*
> * **conf.set("table.name", args[1]);*
> * **try {*
> * **String tblName = conf.get("table.name");*
> * **try {*
> * **HBaseAdmin admin = new HBaseAdmin((HBaseConfiguration) conf);*
> * **if (!admin.tableExists(tblName)) {*
> * **Thread.currentThread().sleep(2000);*
> * **HTableDescriptor tblDesc = new HTableDescriptor(tblName);*
> * **tblDesc.addFamily(new HColumnDescriptor("queries"));*
> * **admin.createTable(tblDesc);*
> * **}*
> * **} catch (TableExistsException e) {*
> * **} catch (InterruptedException e) {*
> * **e.printStackTrace();*
> * **}*
> * **} catch (IOException e) {*
> * **e.printStackTrace();*
> * **}*
> * **Job job = new Job(conf);*
> * **// Job job = new Job(new Configuration());*
> * **job.setJarByClass(FasterMidImport.class);*
> * **job.setJobName("Faster-midImport: import " + conf.get("input.dir")*
> * **+ "into" + conf.get("input.dir"));*
> *
> *
> * **job.setOutputKeyClass(Text.class);*
> * **job.setOutputValueClass(Text.class);*
> * **job.setOutputFormatClass(TableOutputFormat.class);*
> *
> *
> * **job.setInputFormatClass(TextInputFormat.class);*
> * **FileInputFormat.setInputPaths(job, conf.get("input.dir"));*
> *
> *
> * **job.setMapperClass(InnerMap.class);*
> *
> *
> * **job.setMapOutputKeyClass(Text.class);*
> * **job.setMapOutputValueClass(Text.class);*
> *
> *
> * **job.setNumReduceTasks(20);*
> * **//
> //////////////////////////////////////////////////////////////////////////////////////////////////////////////
> *
> * **TableMapReduceUtil.initTableReducerJob(conf.get("table.name"),*
> * **MyTableReducer.class, job);*
> * **try {*
> * **job.submit();*
> * **while (!job.isComplete()) {*
> * **Thread.currentThread().sleep(1000);*
> * **System.out*
> * **.println("Map: " + (job.mapProgress() * 100)*
> * **+ "% ... Reduce: "*
> * **+ (job.reduceProgress() * 100) + "%");*
> *
> *
> * **}*
> * **} catch (InterruptedException e) {*
> * **e.printStackTrace();*
> * **} catch (ClassNotFoundException e) {*
> * **e.printStackTrace();*
> * **}*
> * **}*
> * **static int printUsage() {*
> * **System.err.println("Usage: FasterMidImport <input-dir> <table_name>");*
> * **return -1;*
> * **}*
> * **/***
> * ** * * Run*
> * ** * */*
> * **public int run(String[] args) throws Exception {*
> * **if (args.length < 2) {*
> * **return printUsage();*
> * **}*
> * **this.doJob(args);*
> * **return 0;*
> * **}*
> *
> *
> * **public static int main(String[] args) throws Exception {*
> * **new FasterMidImport().run(args);*
> * **return 0;*
> * **}*
> *}*
>
>
> --
> best wishes.
> steven
>