You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-user@hadoop.apache.org by Mayank Mishra <ma...@gmail.com> on 2016/04/27 16:27:54 UTC
Issue on accessing AVRO format file from HA Cluster.
Hi,
Kindly advise me to resolve the below exception getting while trying to
access the AVRO file on HA cluster from the below code. I have attached
the resource file and pasted too.
*Code: **import* java.io.IOException;
*import* org.apache.avro.file.DataFileReader;
*import* org.apache.avro.file.FileReader;
*import* org.apache.avro.file.SeekableInput;
*import* org.apache.avro.generic.GenericDatumReader;
*import* org.apache.avro.generic.GenericRecord;
*import* org.apache.avro.io.DatumReader;
*import* org.apache.avro.mapred.FsInput;
*import* org.apache.hadoop.conf.Configuration;
*import* org.apache.hadoop.fs.Path;
*public* *static* *void* main(String args[]) *throws* IOException{
Path path = *new* Path(
"hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");
Configuration config = *new* Configuration(); // make this your
*Hadoop* *env* *config*
config.addResource(
"C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
);
config.addResource(
"C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");
//*Metadata* fetch for each record, we sample only one record in
actual implementation.
SeekableInput input = *new* FsInput(path, config); <-Exception
on this line
DatumReader<GenericRecord> reader = *new*
GenericDatumReader<GenericRecord>();
FileReader<GenericRecord> fileReader = DataFileReader.
*openReader*(input, reader);
fileReader.close(); // also closes underlying FsInput
}
*Exception is : *
Exception in thread "main" *java.lang.IllegalArgumentException*:
*java.net.UnknownHostException*: infaqaha
at org.apache.hadoop.security.SecurityUtil.buildTokenService(
*SecurityUtil.java:374*)
at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
*NameNodeProxies.java:312*)
at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
*NameNodeProxies.java:178*)
at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)
at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)
at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
*DistributedFileSystem.java:148*)
at org.apache.hadoop.fs.FileSystem.createFileSystem(
*FileSystem.java:2596*)
at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
*FileSystem.java:2630*)
at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)
at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)
at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)
at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)
at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)
Caused by: *java.net.UnknownHostException*: infaqaha
... 14 more
ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
-2
JDWP exit error AGENT_ERROR_NO_JNI_ENV(183): [util.c:840]
*hdfs -site.xml:*
<?xml version="1.0" encoding="UTF-8"?>
<!--Autogenerated by Cloudera Manager-->
<configuration>
<property>
<name>dfs.nameservices</name>
<value>INFAQAHA</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.INFAQAHA</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
<value>true</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>ivlhdp18.informatica.com:2181</value>
</property>
<property>
<name>dfs.ha.namenodes.INFAQAHA</name>
<value>namenode34,namenode54</value>
</property>
<property>
<name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:8020</value>
</property>
<property>
<name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:8022</value>
</property>
<property>
<name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
<value>ivlhdp18.informatica.com:50470</value>
</property>
<property>
<name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:8020</value>
</property>
<property>
<name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:8022</value>
</property>
<property>
<name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:50070</value>
</property>
<property>
<name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
<value>ivlhdp19.informatica.com:50470</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
</property>
<property>
<name>dfs.client.use.datanode.hostname</name>
<value>false</value>
</property>
<property>
<name>fs.permissions.umask-mode</name>
<value>022</value>
</property>
<property>
<name>dfs.namenode.acls.enabled</name>
<value>false</value>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>false</value>
</property>
<property>
<name>dfs.domain.socket.path</name>
<value>/var/run/hdfs-sockets/dn</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.skip.checksum</name>
<value>false</value>
</property>
<property>
<name>dfs.client.domain.socket.data.traffic</name>
<value>false</value>
</property>
<property>
<name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
</property>
</configuration>
*core-site.xml: *
<?xml version="1.0" encoding="UTF-8"?>
<!--Autogenerated by Cloudera Manager-->
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://INFAQAHA</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>1</value>
</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
</property>
<property>
<name>hadoop.security.authentication</name>
<value>simple</value>
</property>
<property>
<name>hadoop.security.authorization</name>
<value>false</value>
</property>
<property>
<name>hadoop.rpc.protection</name>
<value>authentication</value>
</property>
<property>
<name>hadoop.ssl.require.client.cert</name>
<value>false</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.keystores.factory.class</name>
<value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.server.conf</name>
<value>ssl-server.xml</value>
<final>true</final>
</property>
<property>
<name>hadoop.ssl.client.conf</name>
<value>ssl-client.xml</value>
<final>true</final>
</property>
<property>
<name>hadoop.security.auth_to_local</name>
<value>DEFAULT</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.oozie.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.flume.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.flume.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.HTTP.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hdfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.yarn.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.security.group.mapping</name>
<value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
<property>
<name>hadoop.security.instrumentation.requires.admin</name>
<value>false</value>
</property>
</configuration>
Thanks,
Mayank Mishra
Re: Issue on accessing AVRO format file from HA Cluster.
Posted by Niels Basjes <Ni...@basjes.nl>.
Hi,
You say are on an HA cluster; yet by just looking at the errors I see the
stack being routed through "org.apache.hadoop.hdfs.NameNodeProxies.
createNonHAProxy"
My best guess is that you HA config is incomplete.
Niels Basjes
On Wed, Apr 27, 2016 at 4:27 PM, Mayank Mishra <mayank.mishra.nitb@gmail.com
> wrote:
> Hi,
>
> Kindly advise me to resolve the below exception getting while trying to
> access the AVRO file on HA cluster from the below code. I have attached
> the resource file and pasted too.
>
>
> *Code: **import* java.io.IOException;
>
> *import* org.apache.avro.file.DataFileReader;
>
> *import* org.apache.avro.file.FileReader;
>
> *import* org.apache.avro.file.SeekableInput;
>
> *import* org.apache.avro.generic.GenericDatumReader;
>
> *import* org.apache.avro.generic.GenericRecord;
>
> *import* org.apache.avro.io.DatumReader;
>
> *import* org.apache.avro.mapred.FsInput;
>
> *import* org.apache.hadoop.conf.Configuration;
>
> *import* org.apache.hadoop.fs.Path;
>
> *public* *static* *void* main(String args[]) *throws* IOException{
>
> Path path = *new* Path(
> "hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");
>
> Configuration config = *new* Configuration(); // make this
> your *Hadoop* *env* *config*
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
> );
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");
>
>
>
> //*Metadata* fetch for each record, we sample only one record
> in actual implementation.
>
> SeekableInput input = *new* FsInput(path, config); <-Exception
> on this line
>
> DatumReader<GenericRecord> reader = *new*
> GenericDatumReader<GenericRecord>();
>
> FileReader<GenericRecord> fileReader = DataFileReader.
> *openReader*(input, reader);
>
> fileReader.close(); // also closes underlying FsInput
> }
>
>
>
>
> *Exception is : *
>
> Exception in thread "main" *java.lang.IllegalArgumentException*:
> *java.net.UnknownHostException*: infaqaha
>
> at org.apache.hadoop.security.SecurityUtil.buildTokenService(
> *SecurityUtil.java:374*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
> *NameNodeProxies.java:312*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
> *NameNodeProxies.java:178*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)
>
> at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
> *DistributedFileSystem.java:148*)
>
> at org.apache.hadoop.fs.FileSystem.createFileSystem(
> *FileSystem.java:2596*)
>
> at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
> *FileSystem.java:2630*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)
>
> at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)
>
> at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)
>
> at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)
>
> at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)
>
> Caused by: *java.net.UnknownHostException*: infaqaha
>
> ... 14 more
>
> ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
> -2
>
> JDWP exit error AGENT_ERROR_NO_JNI_ENV(183): [util.c:840]
>
>
>
> *hdfs -site.xml:*
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>dfs.nameservices</name>
>
> <value>INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.failover.proxy.provider.INFAQAHA</name>
>
>
> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
>
> <value>true</value>
>
> </property>
>
> <property>
>
> <name>ha.zookeeper.quorum</name>
>
> <value>ivlhdp18.informatica.com:2181</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.namenodes.INFAQAHA</name>
>
> <value>namenode34,namenode54</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.replication</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>dfs.blocksize</name>
>
> <value>134217728</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.use.datanode.hostname</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>fs.permissions.umask-mode</name>
>
> <value>022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.acls.enabled</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.domain.socket.path</name>
>
> <value>/var/run/hdfs-sockets/dn</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit.skip.checksum</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.domain.socket.data.traffic</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
>
> <value>true</value>
>
> </property>
>
> </configuration>
>
>
>
> *core-site.xml: *
>
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>fs.defaultFS</name>
>
> <value>hdfs://INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>fs.trash.interval</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>io.compression.codecs</name>
>
>
> <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authentication</name>
>
> <value>simple</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authorization</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>hadoop.rpc.protection</name>
>
> <value>authentication</value>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.require.client.cert</name>
>
> <value>false</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.keystores.factory.class</name>
>
> <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.server.conf</name>
>
> <value>ssl-server.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.client.conf</name>
>
> <value>ssl-client.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.security.auth_to_local</name>
>
> <value>DEFAULT</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.group.mapping</name>
>
> <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.instrumentation.requires.admin</name>
>
> <value>false</value>
>
> </property>
>
> </configuration>
>
>
> Thanks,
> Mayank Mishra
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: user-unsubscribe@hadoop.apache.org
> For additional commands, e-mail: user-help@hadoop.apache.org
>
--
Best regards / Met vriendelijke groeten,
Niels Basjes
Re: Issue on accessing AVRO format file from HA Cluster.
Posted by Niels Basjes <Ni...@basjes.nl>.
Hi,
You say are on an HA cluster; yet by just looking at the errors I see the
stack being routed through "org.apache.hadoop.hdfs.NameNodeProxies.
createNonHAProxy"
My best guess is that you HA config is incomplete.
Niels Basjes
On Wed, Apr 27, 2016 at 4:27 PM, Mayank Mishra <mayank.mishra.nitb@gmail.com
> wrote:
> Hi,
>
> Kindly advise me to resolve the below exception getting while trying to
> access the AVRO file on HA cluster from the below code. I have attached
> the resource file and pasted too.
>
>
> *Code: **import* java.io.IOException;
>
> *import* org.apache.avro.file.DataFileReader;
>
> *import* org.apache.avro.file.FileReader;
>
> *import* org.apache.avro.file.SeekableInput;
>
> *import* org.apache.avro.generic.GenericDatumReader;
>
> *import* org.apache.avro.generic.GenericRecord;
>
> *import* org.apache.avro.io.DatumReader;
>
> *import* org.apache.avro.mapred.FsInput;
>
> *import* org.apache.hadoop.conf.Configuration;
>
> *import* org.apache.hadoop.fs.Path;
>
> *public* *static* *void* main(String args[]) *throws* IOException{
>
> Path path = *new* Path(
> "hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");
>
> Configuration config = *new* Configuration(); // make this
> your *Hadoop* *env* *config*
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
> );
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");
>
>
>
> //*Metadata* fetch for each record, we sample only one record
> in actual implementation.
>
> SeekableInput input = *new* FsInput(path, config); <-Exception
> on this line
>
> DatumReader<GenericRecord> reader = *new*
> GenericDatumReader<GenericRecord>();
>
> FileReader<GenericRecord> fileReader = DataFileReader.
> *openReader*(input, reader);
>
> fileReader.close(); // also closes underlying FsInput
> }
>
>
>
>
> *Exception is : *
>
> Exception in thread "main" *java.lang.IllegalArgumentException*:
> *java.net.UnknownHostException*: infaqaha
>
> at org.apache.hadoop.security.SecurityUtil.buildTokenService(
> *SecurityUtil.java:374*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
> *NameNodeProxies.java:312*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
> *NameNodeProxies.java:178*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)
>
> at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
> *DistributedFileSystem.java:148*)
>
> at org.apache.hadoop.fs.FileSystem.createFileSystem(
> *FileSystem.java:2596*)
>
> at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
> *FileSystem.java:2630*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)
>
> at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)
>
> at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)
>
> at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)
>
> at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)
>
> Caused by: *java.net.UnknownHostException*: infaqaha
>
> ... 14 more
>
> ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
> -2
>
> JDWP exit error AGENT_ERROR_NO_JNI_ENV(183): [util.c:840]
>
>
>
> *hdfs -site.xml:*
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>dfs.nameservices</name>
>
> <value>INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.failover.proxy.provider.INFAQAHA</name>
>
>
> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
>
> <value>true</value>
>
> </property>
>
> <property>
>
> <name>ha.zookeeper.quorum</name>
>
> <value>ivlhdp18.informatica.com:2181</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.namenodes.INFAQAHA</name>
>
> <value>namenode34,namenode54</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.replication</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>dfs.blocksize</name>
>
> <value>134217728</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.use.datanode.hostname</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>fs.permissions.umask-mode</name>
>
> <value>022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.acls.enabled</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.domain.socket.path</name>
>
> <value>/var/run/hdfs-sockets/dn</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit.skip.checksum</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.domain.socket.data.traffic</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
>
> <value>true</value>
>
> </property>
>
> </configuration>
>
>
>
> *core-site.xml: *
>
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>fs.defaultFS</name>
>
> <value>hdfs://INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>fs.trash.interval</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>io.compression.codecs</name>
>
>
> <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authentication</name>
>
> <value>simple</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authorization</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>hadoop.rpc.protection</name>
>
> <value>authentication</value>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.require.client.cert</name>
>
> <value>false</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.keystores.factory.class</name>
>
> <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.server.conf</name>
>
> <value>ssl-server.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.client.conf</name>
>
> <value>ssl-client.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.security.auth_to_local</name>
>
> <value>DEFAULT</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.group.mapping</name>
>
> <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.instrumentation.requires.admin</name>
>
> <value>false</value>
>
> </property>
>
> </configuration>
>
>
> Thanks,
> Mayank Mishra
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: user-unsubscribe@hadoop.apache.org
> For additional commands, e-mail: user-help@hadoop.apache.org
>
--
Best regards / Met vriendelijke groeten,
Niels Basjes
Re: Issue on accessing AVRO format file from HA Cluster.
Posted by Niels Basjes <Ni...@basjes.nl>.
Hi,
You say are on an HA cluster; yet by just looking at the errors I see the
stack being routed through "org.apache.hadoop.hdfs.NameNodeProxies.
createNonHAProxy"
My best guess is that you HA config is incomplete.
Niels Basjes
On Wed, Apr 27, 2016 at 4:27 PM, Mayank Mishra <mayank.mishra.nitb@gmail.com
> wrote:
> Hi,
>
> Kindly advise me to resolve the below exception getting while trying to
> access the AVRO file on HA cluster from the below code. I have attached
> the resource file and pasted too.
>
>
> *Code: **import* java.io.IOException;
>
> *import* org.apache.avro.file.DataFileReader;
>
> *import* org.apache.avro.file.FileReader;
>
> *import* org.apache.avro.file.SeekableInput;
>
> *import* org.apache.avro.generic.GenericDatumReader;
>
> *import* org.apache.avro.generic.GenericRecord;
>
> *import* org.apache.avro.io.DatumReader;
>
> *import* org.apache.avro.mapred.FsInput;
>
> *import* org.apache.hadoop.conf.Configuration;
>
> *import* org.apache.hadoop.fs.Path;
>
> *public* *static* *void* main(String args[]) *throws* IOException{
>
> Path path = *new* Path(
> "hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");
>
> Configuration config = *new* Configuration(); // make this
> your *Hadoop* *env* *config*
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
> );
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");
>
>
>
> //*Metadata* fetch for each record, we sample only one record
> in actual implementation.
>
> SeekableInput input = *new* FsInput(path, config); <-Exception
> on this line
>
> DatumReader<GenericRecord> reader = *new*
> GenericDatumReader<GenericRecord>();
>
> FileReader<GenericRecord> fileReader = DataFileReader.
> *openReader*(input, reader);
>
> fileReader.close(); // also closes underlying FsInput
> }
>
>
>
>
> *Exception is : *
>
> Exception in thread "main" *java.lang.IllegalArgumentException*:
> *java.net.UnknownHostException*: infaqaha
>
> at org.apache.hadoop.security.SecurityUtil.buildTokenService(
> *SecurityUtil.java:374*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
> *NameNodeProxies.java:312*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
> *NameNodeProxies.java:178*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)
>
> at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
> *DistributedFileSystem.java:148*)
>
> at org.apache.hadoop.fs.FileSystem.createFileSystem(
> *FileSystem.java:2596*)
>
> at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
> *FileSystem.java:2630*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)
>
> at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)
>
> at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)
>
> at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)
>
> at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)
>
> Caused by: *java.net.UnknownHostException*: infaqaha
>
> ... 14 more
>
> ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
> -2
>
> JDWP exit error AGENT_ERROR_NO_JNI_ENV(183): [util.c:840]
>
>
>
> *hdfs -site.xml:*
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>dfs.nameservices</name>
>
> <value>INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.failover.proxy.provider.INFAQAHA</name>
>
>
> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
>
> <value>true</value>
>
> </property>
>
> <property>
>
> <name>ha.zookeeper.quorum</name>
>
> <value>ivlhdp18.informatica.com:2181</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.namenodes.INFAQAHA</name>
>
> <value>namenode34,namenode54</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.replication</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>dfs.blocksize</name>
>
> <value>134217728</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.use.datanode.hostname</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>fs.permissions.umask-mode</name>
>
> <value>022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.acls.enabled</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.domain.socket.path</name>
>
> <value>/var/run/hdfs-sockets/dn</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit.skip.checksum</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.domain.socket.data.traffic</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
>
> <value>true</value>
>
> </property>
>
> </configuration>
>
>
>
> *core-site.xml: *
>
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>fs.defaultFS</name>
>
> <value>hdfs://INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>fs.trash.interval</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>io.compression.codecs</name>
>
>
> <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authentication</name>
>
> <value>simple</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authorization</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>hadoop.rpc.protection</name>
>
> <value>authentication</value>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.require.client.cert</name>
>
> <value>false</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.keystores.factory.class</name>
>
> <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.server.conf</name>
>
> <value>ssl-server.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.client.conf</name>
>
> <value>ssl-client.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.security.auth_to_local</name>
>
> <value>DEFAULT</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.group.mapping</name>
>
> <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.instrumentation.requires.admin</name>
>
> <value>false</value>
>
> </property>
>
> </configuration>
>
>
> Thanks,
> Mayank Mishra
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: user-unsubscribe@hadoop.apache.org
> For additional commands, e-mail: user-help@hadoop.apache.org
>
--
Best regards / Met vriendelijke groeten,
Niels Basjes
Re: Issue on accessing AVRO format file from HA Cluster.
Posted by Niels Basjes <Ni...@basjes.nl>.
Hi,
You say are on an HA cluster; yet by just looking at the errors I see the
stack being routed through "org.apache.hadoop.hdfs.NameNodeProxies.
createNonHAProxy"
My best guess is that you HA config is incomplete.
Niels Basjes
On Wed, Apr 27, 2016 at 4:27 PM, Mayank Mishra <mayank.mishra.nitb@gmail.com
> wrote:
> Hi,
>
> Kindly advise me to resolve the below exception getting while trying to
> access the AVRO file on HA cluster from the below code. I have attached
> the resource file and pasted too.
>
>
> *Code: **import* java.io.IOException;
>
> *import* org.apache.avro.file.DataFileReader;
>
> *import* org.apache.avro.file.FileReader;
>
> *import* org.apache.avro.file.SeekableInput;
>
> *import* org.apache.avro.generic.GenericDatumReader;
>
> *import* org.apache.avro.generic.GenericRecord;
>
> *import* org.apache.avro.io.DatumReader;
>
> *import* org.apache.avro.mapred.FsInput;
>
> *import* org.apache.hadoop.conf.Configuration;
>
> *import* org.apache.hadoop.fs.Path;
>
> *public* *static* *void* main(String args[]) *throws* IOException{
>
> Path path = *new* Path(
> "hdfs://infaqaha/user/hive/warehouse/avro_table_string/000000_0");
>
> Configuration config = *new* Configuration(); // make this
> your *Hadoop* *env* *config*
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\hadoop-conf\\hdfs-site.xml"
> );
>
> config.addResource(
> "C:\\Users\\mamishra\\Desktop\\hdfs-clientconfig\\core-site");
>
>
>
> //*Metadata* fetch for each record, we sample only one record
> in actual implementation.
>
> SeekableInput input = *new* FsInput(path, config); <-Exception
> on this line
>
> DatumReader<GenericRecord> reader = *new*
> GenericDatumReader<GenericRecord>();
>
> FileReader<GenericRecord> fileReader = DataFileReader.
> *openReader*(input, reader);
>
> fileReader.close(); // also closes underlying FsInput
> }
>
>
>
>
> *Exception is : *
>
> Exception in thread "main" *java.lang.IllegalArgumentException*:
> *java.net.UnknownHostException*: infaqaha
>
> at org.apache.hadoop.security.SecurityUtil.buildTokenService(
> *SecurityUtil.java:374*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createNonHAProxy(
> *NameNodeProxies.java:312*)
>
> at org.apache.hadoop.hdfs.NameNodeProxies.createProxy(
> *NameNodeProxies.java:178*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:662*)
>
> at org.apache.hadoop.hdfs.DFSClient.<init>(*DFSClient.java:606*)
>
> at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(
> *DistributedFileSystem.java:148*)
>
> at org.apache.hadoop.fs.FileSystem.createFileSystem(
> *FileSystem.java:2596*)
>
> at org.apache.hadoop.fs.FileSystem.access$200(*FileSystem.java:91*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(
> *FileSystem.java:2630*)
>
> at org.apache.hadoop.fs.FileSystem$Cache.get(*FileSystem.java:2612*)
>
> at org.apache.hadoop.fs.FileSystem.get(*FileSystem.java:370*)
>
> at org.apache.hadoop.fs.Path.getFileSystem(*Path.java:296*)
>
> at org.apache.avro.mapred.FsInput.<init>(*FsInput.java:38*)
>
> at parquetAndAvro.CopyOfInfaAvro.main(*CopyOfInfaAvro.java:97*)
>
> Caused by: *java.net.UnknownHostException*: infaqaha
>
> ... 14 more
>
> ERROR: JDWP Unable to get JNI 1.2 environment, jvm->GetEnv() return code =
> -2
>
> JDWP exit error AGENT_ERROR_NO_JNI_ENV(183): [util.c:840]
>
>
>
> *hdfs -site.xml:*
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>dfs.nameservices</name>
>
> <value>INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.failover.proxy.provider.INFAQAHA</name>
>
>
> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.automatic-failover.enabled.INFAQAHA</name>
>
> <value>true</value>
>
> </property>
>
> <property>
>
> <name>ha.zookeeper.quorum</name>
>
> <value>ivlhdp18.informatica.com:2181</value>
>
> </property>
>
> <property>
>
> <name>dfs.ha.namenodes.INFAQAHA</name>
>
> <value>namenode34,namenode54</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode34</name>
>
> <value>ivlhdp18.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.rpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8020</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.servicerpc-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:8022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.http-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50070</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.https-address.INFAQAHA.namenode54</name>
>
> <value>ivlhdp19.informatica.com:50470</value>
>
> </property>
>
> <property>
>
> <name>dfs.replication</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>dfs.blocksize</name>
>
> <value>134217728</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.use.datanode.hostname</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>fs.permissions.umask-mode</name>
>
> <value>022</value>
>
> </property>
>
> <property>
>
> <name>dfs.namenode.acls.enabled</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.domain.socket.path</name>
>
> <value>/var/run/hdfs-sockets/dn</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.read.shortcircuit.skip.checksum</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.client.domain.socket.data.traffic</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
>
> <value>true</value>
>
> </property>
>
> </configuration>
>
>
>
> *core-site.xml: *
>
> <?xml version="1.0" encoding="UTF-8"?>
>
>
>
> <!--Autogenerated by Cloudera Manager-->
>
> <configuration>
>
> <property>
>
> <name>fs.defaultFS</name>
>
> <value>hdfs://INFAQAHA</value>
>
> </property>
>
> <property>
>
> <name>fs.trash.interval</name>
>
> <value>1</value>
>
> </property>
>
> <property>
>
> <name>io.compression.codecs</name>
>
>
> <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authentication</name>
>
> <value>simple</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.authorization</name>
>
> <value>false</value>
>
> </property>
>
> <property>
>
> <name>hadoop.rpc.protection</name>
>
> <value>authentication</value>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.require.client.cert</name>
>
> <value>false</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.keystores.factory.class</name>
>
> <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.server.conf</name>
>
> <value>ssl-server.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.ssl.client.conf</name>
>
> <value>ssl-client.xml</value>
>
> <final>true</final>
>
> </property>
>
> <property>
>
> <name>hadoop.security.auth_to_local</name>
>
> <value>DEFAULT</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.oozie.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.mapred.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.flume.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.HTTP.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hive.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hue.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.httpfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.hdfs.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.hosts</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.proxyuser.yarn.groups</name>
>
> <value>*</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.group.mapping</name>
>
> <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
>
> </property>
>
> <property>
>
> <name>hadoop.security.instrumentation.requires.admin</name>
>
> <value>false</value>
>
> </property>
>
> </configuration>
>
>
> Thanks,
> Mayank Mishra
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: user-unsubscribe@hadoop.apache.org
> For additional commands, e-mail: user-help@hadoop.apache.org
>
--
Best regards / Met vriendelijke groeten,
Niels Basjes