You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@storm.apache.org by "Raja.Aravapalli" <Ra...@target.com> on 2015/12/25 05:48:07 UTC
storm-hdfs secure cluster issue
Hi,
I am getting below exception when i am trying to write tuples into HDFS which is in a secured Hadoop cluster. Can someone pls share your thoughts and help me fix the issue
Exception:
---------------
java.lang.RuntimeException: Error preparing HdfsBolt: java.net.UnknownHostException: <HOSTNAME> at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(executor.clj:732) at backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.IllegalArgumentException: java.net.UnknownHostException: <HOSTNAME> at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:374) at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:312) at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:178) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:148) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:107) ... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME> ... 17 more
Code
-------
HdfsBolt hdfsbolt = new HdfsBolt()
.withFsUrl("hdfs://NAMENODESERVICEID")
.withFileNameFormat(fileNameFormat)
.withRecordFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy)
.withConfigKey("hdfs.config”);
Map<String, Object> map = new HashMap<String, Object>();
map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
map.put("hdfs.kerberos.principal", “PRINCIPAL");
Config config = new Config();
config.setDebug(true);
config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
config.put("hdfs.config", map);
StormTopology topology = builder.createTopology();
StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
POM.xml
——————
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>GROUPID</groupId>
<artifactId>ARTIFACTID</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<storm.version>0.9.3.2.2.4.0-2633</storm.version>
<hadoop.version>2.6.0.2.2.4.12-2</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-hdfs</artifactId>
<version>${storm.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<repositories>
<repository>
<id>HDPReleases</id>
<name>HDP Releases</name>
<url>http://repo.hortonworks.com/content/repositories/releases/</url>
<layout>default</layout>
</repository>
<repository>
<id>HDP Jetty Hadoop</id>
<name>HDP Jetty Hadoop</name>
<url>http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
<layout>default</layout>
</repository>
<repository>
<id>confluent</id>
<url>http://packages.confluent.io/maven</url>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.2</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>xml</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
--Raja.
Re: storm-hdfs secure cluster issue
Posted by "Raja.Aravapalli" <Ra...@target.com>.
Thanks a lot Harsha. Your suggestions below worked well.
Regards,
Raja.
From: Harsha <st...@harsha.io>>
Reply-To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Date: Monday, December 28, 2015 at 12:45 AM
To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Subject: Re: storm-hdfs secure cluster issue
Make sure the keytab you are using available on all the supervisor nodes at the same location. Also it should've permissions (atleast read ) for the user who is running the supervisors. Easy way to check this is login as user who is running supervisors and do kinit -k -t keytab.file principal@domain
-Harsha
On Sat, Dec 26, 2015, at 04:46 AM, Raja.Aravapalli wrote:
Hi
I am getting below exception when i am trying to write tuples into HDFS which is in a secured Hadoop cluster. Can someone pls share your thoughts and help me fix the issue
java.lang.RuntimeException: Error preparing HdfsBolt: Login failure for <US...@PRINCIPAL> from keytab <KEYTAB_PATH>: javax.security.auth.login.LoginException: Unable to obtain password from user at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(executor.clj:732) at backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:744) Caused by: java.io.IOException: Login failure for <US...@PRINCIPAL> from keytab <KEYTAB_PATH>: javax.security.auth.login.LoginException: Unable to obtain password from user at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:935) at org.apache.hadoop.security.SecurityUtil.login(SecurityUtil.java:243) at org.apache.hadoop.security.SecurityUtil.login(SecurityUtil.java:207) at org.apache.storm.hdfs.common.security.HdfsSecurityUtil.login(HdfsSecurityUtil.java:56) at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:106) ... 4 more Caused by: javax.security.auth.login.LoginException: Unable to obtain password from user at com.sun.security.auth.module.Krb5LoginModule.promptForPass(Krb5LoginModule.java:856) at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:719) at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:584) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at javax.security.auth.login.LoginContext.invoke(LoginContext.java:784) at javax.security.auth.login.LoginContext.access$000(LoginContext.java:203) at javax.security.auth.login.LoginContext$5.run(LoginContext.java:721) at javax.security.auth.login.LoginContext$5.run(LoginContext.java:719) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.login.LoginContext.invokeCreatorPriv(LoginContext.java:718) at javax.security.auth.login.LoginContext.login(LoginContext.java:590) at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:926) ... 8 more
Regards,
Raja.
From: Raja Aravapalli <Ra...@target.com>>
Reply-To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Date: Saturday, December 26, 2015 at 11:43 AM
To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Subject: Re: storm-hdfs secure cluster issue
Thanks a lot Harsha. After including hdfs-site.xml and core-site.xml the issue is resolved, but facing an other issue.
Regards,
Raja.
From: Harsha <st...@harsha.io>>
Reply-To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Date: Friday, December 25, 2015 at 12:55 PM
To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Subject: Re: storm-hdfs secure cluster issue
you need to package hdfs-site.xml and core-site.xml from your hadoop cluster as part of your topology jar.
http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_storm-user-guide/content/storm-connectors-secure.html
-Harsha
On Thu, Dec 24, 2015, at 08:48 PM, Raja.Aravapalli wrote:
Hi,
I am getting below exception when i am trying to write tuples into HDFS which is in a secured Hadoop cluster. Can someone pls share your thoughts and help me fix the issue
Exception:
---------------
java.lang.RuntimeException: Error preparing HdfsBolt: java.net.UnknownHostException: <HOSTNAME> at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(executor.clj:732) at backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.IllegalArgumentException: java.net.UnknownHostException: <HOSTNAME> at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:374) at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:312) at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:178) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:148) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:107) ... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME> ... 17 more
Code
-------
HdfsBolt hdfsbolt = new HdfsBolt()
.withFsUrl("hdfs://NAMENODESERVICEID")
.withFileNameFormat(fileNameFormat)
.withRecordFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy)
.withConfigKey("hdfs.config”);
Map<String, Object> map = new HashMap<String, Object>();
map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
map.put("hdfs.kerberos.principal", “PRINCIPAL");
Config config = new Config();
config.setDebug(true);
config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
config.put("hdfs.config", map);
StormTopology topology = builder.createTopology();
StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
POM.xml
——————
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>GROUPID</groupId>
<artifactId>ARTIFACTID</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<storm.version>0.9.3.2.2.4.0-2633</storm.version>
<hadoop.version>2.6.0.2.2.4.12-2</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-hdfs</artifactId>
<version>${storm.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<repositories>
<repository>
<id>HDPReleases</id>
<name>HDP Releases</name>
<url>http://repo.hortonworks.com/content/repositories/releases/</url>
<layout>default</layout>
</repository>
<repository>
<id>HDP Jetty Hadoop</id>
<name>HDP Jetty Hadoop</name>
<url>http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
<layout>default</layout>
</repository>
<repository>
<id>confluent</id>
<url>http://packages.confluent.io/maven</url>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.2</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>xml</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
--Raja.
Re: storm-hdfs secure cluster issue
Posted by Harsha <st...@harsha.io>.
Make sure the keytab you are using available on all the supervisor nodes
at the same location. Also it should've permissions (atleast read ) for
the user who is running the supervisors. Easy way to check this is login
as user who is running supervisors and do kinit -k -t keytab.file
principal@domain
-Harsha
On Sat, Dec 26, 2015, at 04:46 AM, Raja.Aravapalli wrote:
>
> Hi
>
> I am getting below exception when i am trying to write tuples into
> HDFS which is in a secured Hadoop cluster. Can someone pls share your
> thoughts and help me fix the issue
>
> java.lang.RuntimeException: Error preparing HdfsBolt: Login failure
for <US...@PRINCIPAL> from keytab <KEYTAB_PATH>:
javax.security.auth.login.LoginException: Unable to obtain password
from user at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(Abstr-
actHdfsBolt.java:111) at backtype.storm.daemon.executor$fn__5265$fn__5-
278.invoke(executor.clj:732) at
backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at
clojure.lang.AFn.run(AFn.java:24) at
java.lang.Thread.run(Thread.java:744) Caused by: java.io.IOException:
Login failure for <US...@PRINCIPAL> from keytab <KEYTAB_PATH>:
javax.security.auth.login.LoginException: Unable to obtain password
from user at org.apache.hadoop.security.UserGroupInformation.loginUser-
FromKeytab(UserGroupInformation.java:935) at
org.apache.hadoop.security.SecurityUtil.login(SecurityUtil.java:243) at
org.apache.hadoop.security.SecurityUtil.login(SecurityUtil.java:207) at
org.apache.storm.hdfs.common.security.HdfsSecurityUtil.login(HdfsSecur-
ityUtil.java:56) at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepar-
e(AbstractHdfsBolt.java:106) ... 4 more Caused by:
javax.security.auth.login.LoginException: Unable to obtain password
from user at com.sun.security.auth.module.Krb5LoginModule.promptForPas-
s(Krb5LoginModule.java:856) at com.sun.security.auth.module.Krb5LoginM-
odule.attemptAuthentication(Krb5LoginModule.java:719) at com.sun.secur-
ity.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:584) at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.ref-
lect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAcc-
essorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606)
at javax.security.auth.login.LoginContext.invoke(LoginContext.java:784)
at javax.security.auth.login.LoginContext.access$000(LoginContext.java-
:203) at
javax.security.auth.login.LoginContext$5.run(LoginContext.java:721) at
javax.security.auth.login.LoginContext$5.run(LoginContext.java:719) at
java.security.AccessController.doPrivileged(Native Method) at javax.se-
curity.auth.login.LoginContext.invokeCreatorPriv(LoginContext.java:718)
at javax.security.auth.login.LoginContext.login(LoginContext.java:590)
at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab-
(UserGroupInformation.java:926) ... 8 more
>
>
> Regards, Raja.
>
>
> *From: *Raja Aravapalli <Ra...@target.com> *Reply-To:
> *"user@storm.apache.org" <us...@storm.apache.org> *Date: *Saturday,
> December 26, 2015 at 11:43 AM *To: *"user@storm.apache.org"
> <us...@storm.apache.org> *Subject: *Re: storm-hdfs secure cluster issue
>
> Thanks a lot Harsha. After including hdfs-site.xml and core-site.xml
> the issue is resolved, but facing an other issue.
>
>
> Regards, Raja.
>
>
> *From: *Harsha <st...@harsha.io> *Reply-To: *"user@storm.apache.org"
> <us...@storm.apache.org> *Date: *Friday, December 25, 2015 at 12:55 PM
> *To: *"user@storm.apache.org" <us...@storm.apache.org> *Subject: *Re:
> storm-hdfs secure cluster issue
>
>
> you need to package hdfs-site.xml and core-site.xml from your hadoop
> cluster as part of your topology jar.
> http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_storm-user-guide/content/storm-connectors-secure.html
>
> -Harsha
>
>
> On Thu, Dec 24, 2015, at 08:48 PM, Raja.Aravapalli wrote:
>>
>>
>> Hi,
>>
>> I am getting below exception when i am trying to write tuples into
>> HDFS which is in a secured Hadoop cluster. Can someone pls share your
>> thoughts and help me fix the issue
>>
>> *Exception: *
>> *---------------*
>> java.lang.RuntimeException:
Error preparing HdfsBolt: java.net.UnknownHostException: <HOSTNAME> at
org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.j-
ava:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(ex-
ecutor.clj:732) at
backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at
clojure.lang.AFn.run(AFn.java:24) at
java.lang.Thread.run(Thread.java:744) Caused by:
java.lang.IllegalArgumentException: java.net.UnknownHostException:
<HOSTNAME> at org.apache.hadoop.security.SecurityUtil.buildTokenServic-
e(SecurityUtil.java:374) at org.apache.hadoop.hdfs.NameNodeProxies.cre-
ateNonHAProxy(NameNodeProxies.java:312) at org.apache.hadoop.hdfs.Name-
NodeProxies.createProxy(NameNodeProxies.java:178) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at org.apa-
che.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem-
.java:148) at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at
org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at
org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at org-
.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java-
:107) ... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME>
... 17 more
>>
>>
>> *Code* *-------* HdfsBolt hdfsbolt = *new *HdfsBolt()
>>
.withFsUrl(*"hdfs://NAMENODESERVICEID"*)
>>
.withFileNameFormat(fileNameFormat)
>>
.withRecordFormat(format)
>>
.withRotationPolicy(rotationPolicy)
>>
.withSyncPolicy(syncPolicy)
>>
.withConfigKey(*"hdfs.config**”*);
>>
>> Map<String, Object> map = new HashMap<String, Object>();
>> map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
>> map.put("hdfs.kerberos.principal", “PRINCIPAL");
>>
>> Config config = new Config(); config.setDebug(true);
>> config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
>> config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
>> config.put("hdfs.config", map);
>>
>> StormTopology topology = builder.createTopology();
>> StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
>>
>> *POM.xml* *——————*
>>
>> <?xml version="1.0" encoding="UTF-8"?> <project xmlns="
>> http://maven.apache.org/POM/4.0.0" xmlns:xsi="
>> http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
>> http://maven.apache.org/xsd/maven-4.0.0.xsd">
>> <modelVersion>4.0.0</modelVersion>
>>
>> <groupId>GROUPID</groupId> <artifactId>ARTIFACTID</artifactId>
>> <version>1.0-SNAPSHOT</version>
>>
>> <properties> <storm.version>0.9.3.2.2.4.0-2633</storm.version>
>> <hadoop.version>2.6.0.2.2.4.12-2</hadoop.version> </properties>
>>
>> <dependencies> <dependency>
>> <groupId>org.apache.storm</groupId> <artifactId>storm-
>> core</artifactId> <version>${storm.version}</version>
>> <scope>provided</scope> </dependency>
>>
>> <dependency> <groupId>org.apache.storm</groupId>
>> <artifactId>storm-hdfs</artifactId>
>> <version>${storm.version}</version> <scope>compile</scope>
>> <exclusions> <exclusion>
>> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-
>> client</artifactId> </exclusion>
>> <exclusion> <groupId>org.apache.hadoop</groupId>
>> <artifactId>hadoop-hdfs</artifactId> </exclusion>
>> <exclusion> <groupId>log4j</groupId>
>> <artifactId>log4j</artifactId> </exclusion>
>> <exclusion> <groupId>org.slf4j</groupId>
>> <artifactId>slf4j-log4j12</artifactId> </exclusion>
>> </exclusions> </dependency>
>>
>> <dependency> <groupId>org.apache.hadoop</groupId>
>> <artifactId>hadoop-hdfs</artifactId>
>> <version>${hadoop.version}</version>
>> <scope>compile</scope> <exclusions>
>> <exclusion> <groupId>log4j</groupId>
>> <artifactId>log4j</artifactId> </exclusion>
>> <exclusion> <groupId>org.slf4j</groupId>
>> <artifactId>slf4j-log4j12</artifactId> </exclusion>
>> </exclusions> </dependency>
>>
>> <dependency> <groupId>org.apache.hadoop</groupId>
>> <artifactId>hadoop-client</artifactId>
>> <version>${hadoop.version}</version>
>> <scope>compile</scope> <exclusions>
>> <exclusion> <groupId>log4j</groupId>
>> <artifactId>log4j</artifactId> </exclusion>
>> <exclusion> <groupId>org.slf4j</groupId>
>> <artifactId>slf4j-log4j12</artifactId> </exclusion>
>> </exclusions> </dependency>
>>
>> </dependencies>
>>
>> <repositories> <repository> <id>HDPReleases</id>
>> <name>HDP Releases</name> <url>
>> http://repo.hortonworks.com/content/repositories/releases/</url>
>> <layout>default</layout> </repository> <repository>
>> <id>HDP Jetty Hadoop</id> <name>HDP Jetty Hadoop</name>
>> <url>
>> http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
>> <layout>default</layout> </repository> <repository>
>> <id>confluent</id> <url>
>> http://packages.confluent.io/maven</url> </repository>
>> </repositories>
>>
>> <build> <plugins> <plugin>
>> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-
>> plugin</artifactId> <version>2.5.1</version>
>> <configuration> <source>1.7</source>
>> <target>1.7</target> </configuration>
>> </plugin> <plugin>
>> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-
>> plugin</artifactId> <version>2.4.2</version>
>> <configuration> <createDependencyReducedPom>true<-
>> /createDependencyReducedPom> </configuration>
>> <executions> <execution>
>> <phase>package</phase> <goals>
>> <goal>shade</goal> </goals>
>> <configuration> <transformers>
>> <transformer implementation="org.apache.maven.plugins.shade.resource-
>> .ServicesResourceTransformer" />
>> <transformer implementation="org.apache.maven.plugins.shade.resource-
>> .ManifestResourceTransformer">
>> <mainClass>xml</mainClass>
>> </transformer> </transformers>
>> <filters> <filter>
>> <artifact>*:*</artifact>
>> <excludes> <exclude>META-
>> INF/*.SF</exclude> <exclude>META-
>> INF/*.DSA</exclude> <exclude>META-
>> INF/*.RSA</exclude> </excludes>
>> </filter> </filters>
>> </configuration> </execution>
>> </executions> </plugin> </plugins> </build>
>> </project>
>>
>>
>> --Raja.
>>
>
>
>
Re: storm-hdfs secure cluster issue
Posted by "Raja.Aravapalli" <Ra...@target.com>.
Hi
I am getting below exception when i am trying to write tuples into HDFS which is in a secured Hadoop cluster. Can someone pls share your thoughts and help me fix the issue
java.lang.RuntimeException: Error preparing HdfsBolt: Login failure for <US...@PRINCIPAL> from keytab <KEYTAB_PATH>: javax.security.auth.login.LoginException: Unable to obtain password from user at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(executor.clj:732) at backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:744) Caused by: java.io.IOException: Login failure for <US...@PRINCIPAL> from keytab <KEYTAB_PATH>: javax.security.auth.login.LoginException: Unable to obtain password from user at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:935) at org.apache.hadoop.security.SecurityUtil.login(SecurityUtil.java:243) at org.apache.hadoop.security.SecurityUtil.login(SecurityUtil.java:207) at org.apache.storm.hdfs.common.security.HdfsSecurityUtil.login(HdfsSecurityUtil.java:56) at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:106) ... 4 more Caused by: javax.security.auth.login.LoginException: Unable to obtain password from user at com.sun.security.auth.module.Krb5LoginModule.promptForPass(Krb5LoginModule.java:856) at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:719) at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:584) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at javax.security.auth.login.LoginContext.invoke(LoginContext.java:784) at javax.security.auth.login.LoginContext.access$000(LoginContext.java:203) at javax.security.auth.login.LoginContext$5.run(LoginContext.java:721) at javax.security.auth.login.LoginContext$5.run(LoginContext.java:719) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.login.LoginContext.invokeCreatorPriv(LoginContext.java:718) at javax.security.auth.login.LoginContext.login(LoginContext.java:590) at org.apache.hadoop.security.UserGroupInformation.loginUserFromKeytab(UserGroupInformation.java:926) ... 8 more
Regards,
Raja.
From: Raja Aravapalli <Ra...@target.com>>
Reply-To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Date: Saturday, December 26, 2015 at 11:43 AM
To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Subject: Re: storm-hdfs secure cluster issue
Thanks a lot Harsha. After including hdfs-site.xml and core-site.xml the issue is resolved, but facing an other issue.
Regards,
Raja.
From: Harsha <st...@harsha.io>>
Reply-To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Date: Friday, December 25, 2015 at 12:55 PM
To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Subject: Re: storm-hdfs secure cluster issue
you need to package hdfs-site.xml and core-site.xml from your hadoop cluster as part of your topology jar.
http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_storm-user-guide/content/storm-connectors-secure.html
-Harsha
On Thu, Dec 24, 2015, at 08:48 PM, Raja.Aravapalli wrote:
Hi,
I am getting below exception when i am trying to write tuples into HDFS which is in a secured Hadoop cluster. Can someone pls share your thoughts and help me fix the issue
Exception:
---------------
java.lang.RuntimeException: Error preparing HdfsBolt: java.net.UnknownHostException: <HOSTNAME> at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(executor.clj:732) at backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.IllegalArgumentException: java.net.UnknownHostException: <HOSTNAME> at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:374) at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:312) at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:178) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:148) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:107) ... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME> ... 17 more
Code
-------
HdfsBolt hdfsbolt = new HdfsBolt()
.withFsUrl("hdfs://NAMENODESERVICEID")
.withFileNameFormat(fileNameFormat)
.withRecordFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy)
.withConfigKey("hdfs.config”);
Map<String, Object> map = new HashMap<String, Object>();
map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
map.put("hdfs.kerberos.principal", “PRINCIPAL");
Config config = new Config();
config.setDebug(true);
config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
config.put("hdfs.config", map);
StormTopology topology = builder.createTopology();
StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
POM.xml
——————
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>GROUPID</groupId>
<artifactId>ARTIFACTID</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<storm.version>0.9.3.2.2.4.0-2633</storm.version>
<hadoop.version>2.6.0.2.2.4.12-2</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-hdfs</artifactId>
<version>${storm.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<repositories>
<repository>
<id>HDPReleases</id>
<name>HDP Releases</name>
<url>http://repo.hortonworks.com/content/repositories/releases/</url>
<layout>default</layout>
</repository>
<repository>
<id>HDP Jetty Hadoop</id>
<name>HDP Jetty Hadoop</name>
<url>http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
<layout>default</layout>
</repository>
<repository>
<id>confluent</id>
<url>http://packages.confluent.io/maven</url>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.2</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>xml</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
--Raja.
Re: storm-hdfs secure cluster issue
Posted by "Raja.Aravapalli" <Ra...@target.com>.
Thanks a lot Harsha. After including hdfs-site.xml and core-site.xml the issue is resolved, but facing an other issue.
Regards,
Raja.
From: Harsha <st...@harsha.io>>
Reply-To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Date: Friday, December 25, 2015 at 12:55 PM
To: "user@storm.apache.org<ma...@storm.apache.org>" <us...@storm.apache.org>>
Subject: Re: storm-hdfs secure cluster issue
you need to package hdfs-site.xml and core-site.xml from your hadoop cluster as part of your topology jar.
http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_storm-user-guide/content/storm-connectors-secure.html
-Harsha
On Thu, Dec 24, 2015, at 08:48 PM, Raja.Aravapalli wrote:
Hi,
I am getting below exception when i am trying to write tuples into HDFS which is in a secured Hadoop cluster. Can someone pls share your thoughts and help me fix the issue
Exception:
---------------
java.lang.RuntimeException: Error preparing HdfsBolt: java.net.UnknownHostException: <HOSTNAME> at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(executor.clj:732) at backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:744) Caused by: java.lang.IllegalArgumentException: java.net.UnknownHostException: <HOSTNAME> at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:374) at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(NameNodeProxies.java:312) at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(NameNodeProxies.java:178) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:148) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java:107) ... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME> ... 17 more
Code
-------
HdfsBolt hdfsbolt = new HdfsBolt()
.withFsUrl("hdfs://NAMENODESERVICEID")
.withFileNameFormat(fileNameFormat)
.withRecordFormat(format)
.withRotationPolicy(rotationPolicy)
.withSyncPolicy(syncPolicy)
.withConfigKey("hdfs.config”);
Map<String, Object> map = new HashMap<String, Object>();
map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
map.put("hdfs.kerberos.principal", “PRINCIPAL");
Config config = new Config();
config.setDebug(true);
config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
config.put("hdfs.config", map);
StormTopology topology = builder.createTopology();
StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
POM.xml
——————
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>GROUPID</groupId>
<artifactId>ARTIFACTID</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<storm.version>0.9.3.2.2.4.0-2633</storm.version>
<hadoop.version>2.6.0.2.2.4.12-2</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-hdfs</artifactId>
<version>${storm.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<repositories>
<repository>
<id>HDPReleases</id>
<name>HDP Releases</name>
<url>http://repo.hortonworks.com/content/repositories/releases/</url>
<layout>default</layout>
</repository>
<repository>
<id>HDP Jetty Hadoop</id>
<name>HDP Jetty Hadoop</name>
<url>http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
<layout>default</layout>
</repository>
<repository>
<id>confluent</id>
<url>http://packages.confluent.io/maven</url>
</repository>
</repositories>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.2</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>xml</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
--Raja.
Re: storm-hdfs secure cluster issue
Posted by Harsha <st...@harsha.io>.
you need to package hdfs-site.xml and core-site.xml from your hadoop
cluster as part of your topology jar.
http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_storm-user-guide/content/storm-connectors-secure.html
-Harsha
On Thu, Dec 24, 2015, at 08:48 PM, Raja.Aravapalli wrote:
>
>
>
Hi,
>
>
I am getting below exception when i am trying to write tuples into HDFS
which is in a secured Hadoop cluster. Can someone pls share your
thoughts and help me fix the issue
>
> *Exception: *
> *---------------*
> java.lang.RuntimeException: Error preparing HdfsBolt:
> java.net.UnknownHostException: <HOSTNAME> at
org.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.j-
ava:111) at backtype.storm.daemon.executor$fn__5265$fn__5278.invoke(ex-
ecutor.clj:732) at
backtype.storm.util$async_loop$fn__551.invoke(util.clj:463) at
clojure.lang.AFn.run(AFn.java:24) at
java.lang.Thread.run(Thread.java:744) Caused by:
java.lang.IllegalArgumentException: java.net.UnknownHostException:
<HOSTNAME> at org.apache.hadoop.security.SecurityUtil.buildTokenServic-
e(SecurityUtil.java:374) at org.apache.hadoop.hdfs.NameNodeProxies.cre-
ateNonHAProxy(NameNodeProxies.java:312) at org.apache.hadoop.hdfs.Name-
NodeProxies.createProxy(NameNodeProxies.java:178) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:665) at
org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:601) at org.apa-
che.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem-
.java:148) at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2619)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91) at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2653)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2635) at
org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370) at
org.apache.storm.hdfs.bolt.HdfsBolt.doPrepare(HdfsBolt.java:87) at org-
.apache.storm.hdfs.bolt.AbstractHdfsBolt.prepare(AbstractHdfsBolt.java-
:107) ... 4 more Caused by: java.net.UnknownHostException: <HOSTNAME>
... 17 more
>
>
> *Code* *-------* HdfsBolt hdfsbolt = *new *HdfsBolt()
>
.withFsUrl(*"hdfs://NAMENODESERVICEID"*)
>
.withFileNameFormat(fileNameFormat)
>
.withRecordFormat(format)
>
.withRotationPolicy(rotationPolicy)
>
.withSyncPolicy(syncPolicy)
>
.withConfigKey(*"hdfs.config**”*);
>
> Map<String, Object> map = new HashMap<String, Object>();
> map.put("hdfs.keytab.file", “PATH_TO_KEYTAB");
> map.put("hdfs.kerberos.principal", “PRINCIPAL");
>
> Config config = new Config(); config.setDebug(true);
> config.put(HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY, “PATH_TO_KEYTAB");
> config.put(HdfsSecurityUtil.STORM_USER_NAME_KEY, "PRINCIPAL");
> config.put("hdfs.config", map);
>
> StormTopology topology = builder.createTopology();
> StormSubmitter.submitTopology(“<TOPOLOGY_NAME>", config, topology);
>
> *POM.xml* *——————*
>
> <?xml version="1.0" encoding="UTF-8"?> <project xmlns="
> http://maven.apache.org/POM/4.0.0" xmlns:xsi="
> http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="
> http://maven.apache.org/POM/4.0.0
> http://maven.apache.org/xsd/maven-4.0.0.xsd">
> <modelVersion>4.0.0</modelVersion>
>
> <groupId>GROUPID</groupId> <artifactId>ARTIFACTID</artifactId>
> <version>1.0-SNAPSHOT</version>
>
> <properties> <storm.version>0.9.3.2.2.4.0-2633</storm.version>
> <hadoop.version>2.6.0.2.2.4.12-2</hadoop.version> </properties>
>
> <dependencies> <dependency>
> <groupId>org.apache.storm</groupId> <artifactId>storm-
> core</artifactId> <version>${storm.version}</version>
> <scope>provided</scope> </dependency>
>
> <dependency> <groupId>org.apache.storm</groupId>
> <artifactId>storm-hdfs</artifactId>
> <version>${storm.version}</version> <scope>compile</scope>
> <exclusions> <exclusion>
> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-
> client</artifactId> </exclusion>
> <exclusion> <groupId>org.apache.hadoop</groupId>
> <artifactId>hadoop-hdfs</artifactId> </exclusion>
> <exclusion> <groupId>log4j</groupId>
> <artifactId>log4j</artifactId> </exclusion>
> <exclusion> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-log4j12</artifactId> </exclusion>
> </exclusions> </dependency>
>
> <dependency> <groupId>org.apache.hadoop</groupId>
> <artifactId>hadoop-hdfs</artifactId>
> <version>${hadoop.version}</version> <scope>compile</scope>
> <exclusions> <exclusion>
> <groupId>log4j</groupId>
> <artifactId>log4j</artifactId> </exclusion>
> <exclusion> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-log4j12</artifactId> </exclusion>
> </exclusions> </dependency>
>
> <dependency> <groupId>org.apache.hadoop</groupId>
> <artifactId>hadoop-client</artifactId>
> <version>${hadoop.version}</version> <scope>compile</scope>
> <exclusions> <exclusion>
> <groupId>log4j</groupId>
> <artifactId>log4j</artifactId> </exclusion>
> <exclusion> <groupId>org.slf4j</groupId>
> <artifactId>slf4j-log4j12</artifactId> </exclusion>
> </exclusions> </dependency>
>
> </dependencies>
>
> <repositories> <repository> <id>HDPReleases</id>
> <name>HDP Releases</name> <url>
> http://repo.hortonworks.com/content/repositories/releases/</url>
> <layout>default</layout> </repository> <repository>
> <id>HDP Jetty Hadoop</id> <name>HDP Jetty Hadoop</name>
> <url>
> http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
> <layout>default</layout> </repository> <repository>
> <id>confluent</id> <url>
> http://packages.confluent.io/maven</url> </repository>
> </repositories>
>
> <build> <plugins> <plugin>
> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-
> plugin</artifactId> <version>2.5.1</version>
> <configuration> <source>1.7</source>
> <target>1.7</target> </configuration>
> </plugin> <plugin>
> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-
> plugin</artifactId> <version>2.4.2</version>
> <configuration> <createDependencyReducedPom>true</-
> createDependencyReducedPom> </configuration>
> <executions> <execution>
> <phase>package</phase> <goals>
> <goal>shade</goal> </goals>
> <configuration> <transformers>
> <transformer implementation="org.apache.maven.plugins.shade.resource.-
> ServicesResourceTransformer" />
> <transformer implementation="org.apache.maven.plugins.shade.resource.-
> ManifestResourceTransformer">
> <mainClass>xml</mainClass>
> </transformer> </transformers>
> <filters> <filter>
> <artifact>*:*</artifact> <excludes>
> <exclude>META-INF/*.SF</exclude>
> <exclude>META-INF/*.DSA</exclude>
> <exclude>META-INF/*.RSA</exclude>
> </excludes> </filter>
> </filters> </configuration>
> </execution> </executions> </plugin>
> </plugins> </build> </project>
>
>
> --Raja.
>