You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@hadoop.apache.org by Michael Gao <mi...@hotmail.com> on 2022/05/21 15:26:59 UTC
How to create object Configuration?
Hi all
I try to build some application on windows to access my hdfs on linux.
Here is my code:
package com.michael.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
public class HdfsClient {
@Test
public void testmkdir() throws IOException, URISyntaxException, InterruptedException {
URI url = new URI("hdfs://192.18.3.31:8020");
Configuration conf = new Configuration();
String user="michael";
FileSystem fs = FileSystem.get(url,conf,user);
// do some thing
fs.close();
}
}
And here is my porm.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.michael</groupId>
<artifactId>hdfs</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>18</maven.compiler.source>
<maven.compiler.target>18</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.3.2</version>
</dependency>
</dependencies>
</project>
But each time I fail at
Configuration conf = new Configuration();
Here is error info:
java.lang.NoClassDefFoundError: org/apache/hadoop/shaded/com/ctc/wstx/io/InputBootstrapper
at com.michael.hdfs.HdfsClient.testmkdir(HdfsClient.java:18)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base/java.lang.reflect.Method.invoke(Method.java:577)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69)
at com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38)
at com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11)
at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35)
at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:235)
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:54)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.shaded.com.ctc.wstx.io.InputBootstrapper
at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521)
... 26 more
What's the problem and how to fix it?
Thanks!
Re: How to create object Configuration?
Posted by Ayush Saxena <ay...@gmail.com>.
Hi Michael,
Can you change the hadoop version from 3.3.2 to 3.3.3 and then try, Most
probably this is due to missing shaded classes in the 3.3.2 version from
the hadoop-client jar.
-Ayush
On Sat, 21 May 2022 at 20:57, Michael Gao <mi...@hotmail.com> wrote:
> Hi all
>
> I try to build some application on windows to access my hdfs on linux.
>
>
>
> Here is my code:
>
> package com.michael.hdfs;
>
>
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.fs.FileSystem;
>
> import org.junit.Test;
> import java.io.IOException;
> import java.net.URI;
> import java.net.URISyntaxException;
>
>
> public class HdfsClient {
> @Test
> public void testmkdir() throws IOException, URISyntaxException, InterruptedException {
> URI url = new URI("hdfs://192.18.3.31:8020");
> Configuration conf = new Configuration();
> String user="michael";
> FileSystem fs = FileSystem.*get*(url,conf,user);
>
> // do some thing
>
> fs.close();
> }
> }
>
>
>
> And here is my porm.xml
>
> <?xml version="1.0" encoding="UTF-8"?>
> <project xmlns="http://maven.apache.org/POM/4.0.0"
> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
> <modelVersion>4.0.0</modelVersion>
>
> <groupId>org.michael</groupId>
> <artifactId>hdfs</artifactId>
> <version>1.0-SNAPSHOT</version>
> <properties>
> <maven.compiler.source>18</maven.compiler.source>
> <maven.compiler.target>18</maven.compiler.target>
> </properties>
> <dependencies>
> <dependency>
> <groupId>junit</groupId>
> <artifactId>junit</artifactId>
> <version>4.13.2</version>
> </dependency>
> <dependency>
> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-log4j12</artifactId>
> <version>1.7.36</version>
> </dependency>
> <dependency>
> <groupId>org.apache.hadoop</groupId>
> <artifactId>hadoop-client</artifactId>
> <version>3.3.2</version>
> </dependency>
> </dependencies>
> </project>
>
>
>
> But each time I fail at
>
> Configuration conf = new Configuration();
>
>
>
> Here is error info:
>
> *java.lang.NoClassDefFoundError:
> org/apache/hadoop/shaded/com/ctc/wstx/io/InputBootstrapper*
>
>
>
> * at com.michael.hdfs.HdfsClient.testmkdir(HdfsClient.java:18)*
>
> * at
> java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)*
>
> * at java.base/java.lang.reflect.Method.invoke(Method.java:577)*
>
> * at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)*
>
> * at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)*
>
> * at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)*
>
> * at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)*
>
> * at
> org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)*
>
> * at
> org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)*
>
> * at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)*
>
> * at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)*
>
> * at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)*
>
> * at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)*
>
> * at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)*
>
> * at
> org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)*
>
> * at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)*
>
> * at
> org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)*
>
> * at
> org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)*
>
> * at org.junit.runners.ParentRunner.run(ParentRunner.java:413)*
>
> * at org.junit.runner.JUnitCore.run(JUnitCore.java:137)*
>
> * at
> com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69)*
>
> * at
> com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38)*
>
> * at
> com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11)*
>
> * at
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35)*
>
> * at
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:235)*
>
> * at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:54)*
>
> *Caused by: java.lang.ClassNotFoundException:
> org.apache.hadoop.shaded.com.ctc.wstx.io.InputBootstrapper*
>
> * at
> java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)*
>
> * at
> java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)*
>
> * at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521)*
>
> * ... 26 more*
>
>
>
> What’s the problem and how to fix it?
>
> Thanks!
>
>
>
>
>