You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@kylin.apache.org by Sivakumar Venkataraman <gu...@gmail.com> on 2017/01/06 06:38:28 UTC
#3 Step Name: Extract Fact Table Distinct Error
We are new to kylin. We are trying to build a sample cube.
While in Step 3, the following exception is thrown.
java.lang.RuntimeException: java.io.IOException:
org.apache.thrift.transport.TTransportException
at
org.apache.kylin.source.hive.HiveMRInput$HiveTableInputFormat.configureJob(HiveMRInput.java:94)
at
org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob.setupMapper(FactDistinctColumnsJob.java:123)
at
org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob.run(FactDistinctColumnsJob.java:103)
at org.apache.kylin.engine.mr.MRUtil.runMRJob(MRUtil.java:88)
at
org.apache.kylin.engine.mr.common.MapReduceExecutable.doWork(MapReduceExecutable.java:120)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:113)
at
org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:57)
at
org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:113)
at
org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:136)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException:
org.apache.thrift.transport.TTransportException
at
org.apache.hive.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:97)
at
org.apache.hive.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:51)
at
org.apache.kylin.source.hive.HiveMRInput$HiveTableInputFormat.configureJob(HiveMRInput.java:89)
... 11 more
Caused by: org.apache.thrift.transport.TTransportException
at
org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132)
at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86)
at
org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429)
at
org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318)
at
org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_table(ThriftHiveMetastore.java:1260)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_table(ThriftHiveMetastore.java:1246)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTable(HiveMetaStoreClient.java:1349)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:152)
at com.sun.proxy.$Proxy45.getTable(Unknown Source)
at org.apache.hive.hcatalog.common.HCatUtil.getTable(HCatUtil.java:180)
at
org.apache.hive.hcatalog.mapreduce.InitializeInput.getInputJobInfo(InitializeInput.java:105)
at
org.apache.hive.hcatalog.mapreduce.InitializeInput.setInput(InitializeInput.java:86)
at
org.apache.hive.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:95)
... 13 more
result code:2
Any help is appreciated !
--
View this message in context: http://apache-kylin.74782.x6.nabble.com/3-Step-Name-Extract-Fact-Table-Distinct-Error-tp6864.html
Sent from the Apache Kylin mailing list archive at Nabble.com.
Re: #3 Step Name: Extract Fact Table Distinct Error
Posted by Sivakumar Venkataraman <gu...@gmail.com>.
<?xml version="1.0" encoding="UTF-8" ?>
- <configuration>
- <property>
<name>hive.server2.authentication</name>
<value>KERBEROS</value>
</property>
- <property>
<name>related.hbase.service.name</name>
<value>HBase</value>
</property>
- <property>
<name>hive.metastore.uris</name>
<value>thrift://x.x.x.104:21088,thrift://x.x.x.103:21088</value>
</property>
- <property>
<name>hive.server2.authentication.kerberos.principal</name>
<value>hive/hadoop.hadoop.com@HADOOP.COM</value>
</property>
- <property>
<name>hive.server2.authentication.kerberos.keytab</name>
<value>/opt/huawei/Bigdata/FusionInsight/FusionInsight-Hive-1.3.0/hive-1.3.0/bin/hive.keytab</value>
</property>
- <property>
<name>hive.metastore.kerberos.principal</name>
<value>hive/hadoop.hadoop.com@HADOOP.COM</value>
</property>
- <property>
<name>hive.security.authorization.enabled</name>
<value>false</value>
</property>
- <property>
<name>hive.server2.thrift.sasl.qop</name>
<value>auth-conf</value>
</property>
- <property>
<name>hive.server2.enable.doAs</name>
<value>true</value>
</property>
- <property>
<name>hive.mapred.reduce.tasks.speculative.execution</name>
<value>false</value>
</property>
</configuration>
and metastore configuration is as
<?xml version="1.0" encoding="UTF-8" ?>
- <configuration>
- <property>
<name>hive.metastore.db.ssl.enabled</name>
<value>false</value>
</property>
- <property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>org.postgresql.Driver</value>
</property>
- <property>
<name>hive.metastore.blacklist.keep.times</name>
<value>10000</value>
</property>
- <property>
<name>hive.metastore.sasl.enabled</name>
<value>true</value>
</property>
- <property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive/warehouse</value>
</property>
- <property>
<name>hive.security.acl.flag</name>
<value>HIVE_INTERNAL_USE</value>
</property>
- <property>
<name>hive.metastore.thrift.sasl.qop</name>
<value>auth-conf</value>
</property>
- <property>
<name>javax.jdo.option.ConnectionPassword</name>
<value> encrypted value removed for this post </value>
</property>
- <property>
<name>hive.keychain.dir</name>
<value>/user/hive/keychain</value>
</property>
- <property>
<name>dfs.namenode.acls.enabled</name>
<value>true</value>
</property>
- <property>
<name>javax.jdo.dbuser.decode.enable</name>
<value>true</value>
</property>
- <property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:postgresql://x.x.x.204:20051/hivemeta?socketTimeout=60</value>
</property>
- <property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>hive</value>
</property>
- <property>
<name>hive.exec.scratchdir</name>
<value>/tmp/hive-scratch</value>
</property>
</configuration>
--
View this message in context: http://apache-kylin.74782.x6.nabble.com/3-Step-Name-Extract-Fact-Table-Distinct-Error-tp6864p6884.html
Sent from the Apache Kylin mailing list archive at Nabble.com.
Re: #3 Step Name: Extract Fact Table Distinct Error
Posted by Sivakumar Venkataraman <gu...@gmail.com>.
Yes itpicked up the right hive-site.xml and hive libs are present.
it is retruning the right path. But some jars which are in the
/home/kylin/hadoopclient/Hive/HCatalog/lib and
/home/kylin/hadoopclient/Hive/Beeline/lib are not getting loaded looks like
not sure though!
--
View this message in context: http://apache-kylin.74782.x6.nabble.com/3-Step-Name-Extract-Fact-Table-Distinct-Error-tp6864p6885.html
Sent from the Apache Kylin mailing list archive at Nabble.com.
Re: #3 Step Name: Extract Fact Table Distinct Error
Posted by Li Yang <li...@apache.org>.
Double check kylin picked up the right hive-site.xml and hive libs.
Checkout "bin/find-hive-dependencies.sh", make sure what it returns matches
your expectation.
On Sat, Jan 7, 2017 at 6:42 PM, Cheng Wang <ch...@kyligence.io> wrote:
> It looks like hive configuration is incorrect, can you please provide your
> cluster environment info?
>
> 发自我的 iPhone
>
> > 在 2017年1月7日，下午6:18，Sivakumar Venkataraman <gu...@gmail.com> 写道：
> >
> > We are new to kylin. We are trying to build a sample cube.
> > While in Step 3, the following exception is thrown.
> >
> >
> > java.lang.RuntimeException: java.io.IOException:
> > org.apache.thrift.transport.TTransportException
> > at
> > org.apache.kylin.source.hive.HiveMRInput$HiveTableInputFormat.
> configureJob(HiveMRInput.java:94)
> > at
> > org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob.setupMapper(
> FactDistinctColumnsJob.java:123)
> > at
> > org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob.
> run(FactDistinctColumnsJob.java:103)
> > at org.apache.kylin.engine.mr.MRUtil.runMRJob(MRUtil.java:88)
> > at
> > org.apache.kylin.engine.mr.common.MapReduceExecutable.
> doWork(MapReduceExecutable.java:120)
> > at
> > org.apache.kylin.job.execution.AbstractExecutable.
> execute(AbstractExecutable.java:113)
> > at
> > org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(
> DefaultChainedExecutable.java:57)
> > at
> > org.apache.kylin.job.execution.AbstractExecutable.
> execute(AbstractExecutable.java:113)
> > at
> > org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(
> DefaultScheduler.java:136)
> > at
> > java.util.concurrent.ThreadPoolExecutor.runWorker(
> ThreadPoolExecutor.java:1142)
> > at
> > java.util.concurrent.ThreadPoolExecutor$Worker.run(
> ThreadPoolExecutor.java:617)
> > at java.lang.Thread.run(Thread.java:745)
> > Caused by: java.io.IOException:
> > org.apache.thrift.transport.TTransportException
> > at
> > org.apache.hive.hcatalog.mapreduce.HCatInputFormat.
> setInput(HCatInputFormat.java:97)
> > at
> > org.apache.hive.hcatalog.mapreduce.HCatInputFormat.
> setInput(HCatInputFormat.java:51)
> > at
> > org.apache.kylin.source.hive.HiveMRInput$HiveTableInputFormat.
> configureJob(HiveMRInput.java:89)
> > ... 11 more
> > Caused by: org.apache.thrift.transport.TTransportException
> > at
> > org.apache.thrift.transport.TIOStreamTransport.read(
> TIOStreamTransport.java:132)
> > at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86)
> > at
> > org.apache.thrift.protocol.TBinaryProtocol.readAll(
> TBinaryProtocol.java:429)
> > at
> > org.apache.thrift.protocol.TBinaryProtocol.readI32(
> TBinaryProtocol.java:318)
> > at
> > org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(
> TBinaryProtocol.java:219)
> > at org.apache.thrift.TServiceClient.receiveBase(
> TServiceClient.java:77)
> > at
> > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.
> recv_get_table(ThriftHiveMetastore.java:1260)
> > at
> > org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.
> get_table(ThriftHiveMetastore.java:1246)
> > at
> > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.
> getTable(HiveMetaStoreClient.java:1349)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at
> > sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:62)
> > at
> > sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
> > at java.lang.reflect.Method.invoke(Method.java:498)
> > at
> > org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(
> RetryingMetaStoreClient.java:152)
> > at com.sun.proxy.$Proxy45.getTable(Unknown Source)
> > at org.apache.hive.hcatalog.common.HCatUtil.getTable(
> HCatUtil.java:180)
> > at
> > org.apache.hive.hcatalog.mapreduce.InitializeInput.getInputJobInfo(
> InitializeInput.java:105)
> > at
> > org.apache.hive.hcatalog.mapreduce.InitializeInput.
> setInput(InitializeInput.java:86)
> > at
> > org.apache.hive.hcatalog.mapreduce.HCatInputFormat.
> setInput(HCatInputFormat.java:95)
> > ... 13 more
> >
> > result code:2
> >
> >
> > Any help is appreciated !
> >
> > --
> > View this message in context: http://apache-kylin.74782.x6.
> nabble.com/3-Step-Name-Extract-Fact-Table-Distinct-Error-tp6864.html
> > Sent from the Apache Kylin mailing list archive at Nabble.com.
>
Re: #3 Step Name: Extract Fact Table Distinct Error
Posted by Cheng Wang <ch...@kyligence.io>.
It looks like hive configuration is incorrect, can you please provide your cluster environment info?
发自我的 iPhone
> 在 2017年1月7日，下午6:18，Sivakumar Venkataraman <gu...@gmail.com> 写道：
>
> We are new to kylin. We are trying to build a sample cube.
> While in Step 3, the following exception is thrown.
>
>
> java.lang.RuntimeException: java.io.IOException:
> org.apache.thrift.transport.TTransportException
> at
> org.apache.kylin.source.hive.HiveMRInput$HiveTableInputFormat.configureJob(HiveMRInput.java:94)
> at
> org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob.setupMapper(FactDistinctColumnsJob.java:123)
> at
> org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob.run(FactDistinctColumnsJob.java:103)
> at org.apache.kylin.engine.mr.MRUtil.runMRJob(MRUtil.java:88)
> at
> org.apache.kylin.engine.mr.common.MapReduceExecutable.doWork(MapReduceExecutable.java:120)
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:113)
> at
> org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:57)
> at
> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:113)
> at
> org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:136)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.io.IOException:
> org.apache.thrift.transport.TTransportException
> at
> org.apache.hive.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:97)
> at
> org.apache.hive.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:51)
> at
> org.apache.kylin.source.hive.HiveMRInput$HiveTableInputFormat.configureJob(HiveMRInput.java:89)
> ... 11 more
> Caused by: org.apache.thrift.transport.TTransportException
> at
> org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132)
> at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86)
> at
> org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429)
> at
> org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318)
> at
> org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_table(ThriftHiveMetastore.java:1260)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_table(ThriftHiveMetastore.java:1246)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTable(HiveMetaStoreClient.java:1349)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:152)
> at com.sun.proxy.$Proxy45.getTable(Unknown Source)
> at org.apache.hive.hcatalog.common.HCatUtil.getTable(HCatUtil.java:180)
> at
> org.apache.hive.hcatalog.mapreduce.InitializeInput.getInputJobInfo(InitializeInput.java:105)
> at
> org.apache.hive.hcatalog.mapreduce.InitializeInput.setInput(InitializeInput.java:86)
> at
> org.apache.hive.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:95)
> ... 13 more
>
> result code:2
>
>
> Any help is appreciated !
>
> --
> View this message in context: http://apache-kylin.74782.x6.nabble.com/3-Step-Name-Extract-Fact-Table-Distinct-Error-tp6864.html
> Sent from the Apache Kylin mailing list archive at Nabble.com.