You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hadoop.apache.org by xeonmailinglist <xe...@gmail.com> on 2015/02/03 15:58:16 UTC
Pass lib jars when invoking an hadoop program
Hi,
I am trying to run |distcp| using a java class, but I get the error of
class not found |DistCpOptions|. I have used the argument |-libjars
./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar file,
but it seems that is not right. How I pass the lib properly?
Output:
|vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
-libjars
./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
/input1
/outputmp
/output1
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
|
My class:
|public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length < 2) {
System.err.println("Usage: wordcount <in> [<in>...] <out>");
System.exit(2);
}
System.out.println(conf.toString());
for (int i = 0; i < args.length; i++) {
System.out.println(args[i]);
}
// distcp
String proto = "webhdfs://";
String src = "hadoop-coc-1/input1";
String dest = "hadoop-coc-2/input1";
List<Path> lsrc = new ArrayList<Path>();
lsrc.add(new Path(src));
DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
DistCp distcp = new DistCp(new Configuration(), options);
distcp.execute();
}
|
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here's the solution:
```
vagrant@hadoop-coc-1:~/Programs/hadoop$ export
HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop
jar wordcount.jar -libjars
$HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1
/outputmp /output1
```
On 03-02-2015 14:58, xeonmailinglist wrote:
>
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here's the solution:
```
vagrant@hadoop-coc-1:~/Programs/hadoop$ export
HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop
jar wordcount.jar -libjars
$HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1
/outputmp /output1
```
On 03-02-2015 14:58, xeonmailinglist wrote:
>
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here’s the solution:
|vagrant@hadoop-coc-1:~/Programs/hadoop$ export HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop jar wordcount.jar -libjars $HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
|
On 03-02-2015 14:58, xeonmailinglist wrote:
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here's the solution:
```
vagrant@hadoop-coc-1:~/Programs/hadoop$ export
HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop
jar wordcount.jar -libjars
$HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1
/outputmp /output1
```
On 03-02-2015 14:58, xeonmailinglist wrote:
>
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here’s the solution:
|vagrant@hadoop-coc-1:~/Programs/hadoop$ export HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop jar wordcount.jar -libjars $HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
|
On 03-02-2015 14:58, xeonmailinglist wrote:
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here’s the solution:
|vagrant@hadoop-coc-1:~/Programs/hadoop$ export HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop jar wordcount.jar -libjars $HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
|
On 03-02-2015 14:58, xeonmailinglist wrote:
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here's the solution:
```
vagrant@hadoop-coc-1:~/Programs/hadoop$ export
HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop
jar wordcount.jar -libjars
$HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1
/outputmp /output1
```
On 03-02-2015 14:58, xeonmailinglist wrote:
>
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>
Re: Pass lib jars when invoking an hadoop program
Posted by xeonmailinglist <xe...@gmail.com>.
Got it. Here’s the solution:
|vagrant@hadoop-coc-1:~/Programs/hadoop$ export HADOOP_CLASSPATH=share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar; hadoop jar wordcount.jar -libjars $HADOOP_HOME/share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
|
On 03-02-2015 14:58, xeonmailinglist wrote:
> Hi,
>
> I am trying to run |distcp| using a java class, but I get the error of
> class not found |DistCpOptions|. I have used the argument |-libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar| to pass the jar
> file, but it seems that is not right. How I pass the lib properly?
>
> Output:
>
> |vagrant@hadoop-coc-1:~/Programs/hadoop$ hadoop jar wordcount.jar -libjars ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar /input1 /outputmp /output1
> Configuration: core-default.xml, core-site.xml, mapred-default.xml, mapred-site.xml, yarn-default.xml, yarn-site.xml, hdfs-default.xml, hdfs-site.xml
> -libjars
> ./share/hadoop/tools/lib/hadoop-distcp-2.6.0.jar
> /input1
> /outputmp
> /output1
> Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/tools/DistCpOptions
> at org.apache.hadoop.mapred.examples.WordCount.main(WordCount.java:101)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.tools.DistCpOptions
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> |
>
> My class:
>
> |public static void main(String[] args) throws Exception {
> Configuration conf = new Configuration();
>
> String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
> if (otherArgs.length < 2) {
> System.err.println("Usage: wordcount <in> [<in>...] <out>");
> System.exit(2);
> }
>
> System.out.println(conf.toString());
> for (int i = 0; i < args.length; i++) {
> System.out.println(args[i]);
> }
>
> // distcp
> String proto = "webhdfs://";
> String src = "hadoop-coc-1/input1";
> String dest = "hadoop-coc-2/input1";
> List<Path> lsrc = new ArrayList<Path>();
> lsrc.add(new Path(src));
> DistCpOptions options = new DistCpOptions(lsrc, new Path(dest));
> DistCp distcp = new DistCp(new Configuration(), options);
> distcp.execute();
> }
> |
>