You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by "konglei (JIRA)" <ji...@apache.org> on 2016/12/22 14:39:58 UTC
[jira] [Issue Comment Deleted] (KYLIN-2311) Build Cube
[ https://issues.apache.org/jira/browse/KYLIN-2311?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
konglei updated KYLIN-2311:
---------------------------
Comment: was deleted
(was: [datahadoop@hadoop11 conf]$ cat kylin_job_conf_inmem.xml
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<property>
<name>mapreduce.job.split.metainfo.maxsize</name>
<value>-1</value>
<description>The maximum permissible size of the split metainfo file.
The JobTracker won't attempt to read split metainfo files bigger than
the configured value. No limits if set to -1.
</description>
</property>
<property>
<name>mapred.compress.map.output</name>
<value>true</value>
<description>Compress map outputs</description>
</property>
<!--
The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
-->
<!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
<!--
The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
-->
<!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
<description>The compression type to use for job outputs</description>
</property>
<property>
<name>mapreduce.job.max.split.locations</name>
<value>2000</value>
<description>No description</description>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
<description>Block replication</description>
</property>
<property>
<name>mapred.task.timeout</name>
<value>3600000</value>
<description>Set task timeout to 1 hour</description>
</property>
<!--Additional config for in-mem cubing, giving mapper more memory -->
<property>
<name>mapreduce.map.memory.mb</name>
<value>3072</value>
<description></description>
</property>
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx2700m</value>
<description></description>
</property>
<property>
<name>mapreduce.task.io.sort.mb</name>
<value>200</value>
<description></description>
</property>
</configuration>
[datahadoop@hadoop11 conf]$ cat kylin_job_conf
kylin_job_conf_inmem.xml kylin_job_conf.xml
[datahadoop@hadoop11 conf]$ cat kylin_job_conf.xml
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<property>
<name>mapreduce.job.split.metainfo.maxsize</name>
<value>-1</value>
<description>The maximum permissible size of the split metainfo file.
The JobTracker won't attempt to read split metainfo files bigger than
the configured value. No limits if set to -1.
</description>
</property>
<property>
<name>mapred.compress.map.output</name>
<value>true</value>
<description>Compress map outputs</description>
</property>
<!--
The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
-->
<!--
<property>
<name>mapred.map.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for map outputs
</description>
</property>
-->
<property>
<name>mapred.output.compress</name>
<value>true</value>
<description>Compress the output of a MapReduce job</description>
</property>
<!--
The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
-->
<!--
<property>
<name>mapred.output.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description>The compression codec to use for job outputs
</description>
</property>
-->
<property>
<name>mapred.output.compression.type</name>
<value>BLOCK</value>
<description>The compression type to use for job outputs</description>
</property>
<property>
<name>mapreduce.job.max.split.locations</name>
<value>2000</value>
<description>No description</description>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
<description>Block replication</description>
</property>
<property>
<name>mapred.task.timeout</name>
<value>3600000</value>
<description>Set task timeout to 1 hour</description>
</property>
</configuration>
)
> Build Cube
> ----------
>
> Key: KYLIN-2311
> URL: https://issues.apache.org/jira/browse/KYLIN-2311
> Project: Kylin
> Issue Type: Test
> Components: Job Engine
> Affects Versions: v1.6.0
> Environment: hadoop:cdh 5.7.0
> kylin:apache-kylin-1.6.0-cdh5.7-bin.tar.gz
> system:centos 6.8
> Reporter: konglei
> Assignee: Dong Li
> Priority: Critical
> Fix For: v1.6.0
>
>
> Error: java.io.IOException: Failed to build cube in mapper 1 at org.apache.kylin.engine.mr.steps.InMemCuboidMapper.cleanup(InMemCuboidMapper.java:145) at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:148) at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1693) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: java.util.concurrent.ExecutionException: java.lang.RuntimeException: java.io.IOException: java.io.IOException: java.lang.IllegalStateException at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:188) at org.apache.kylin.engine.mr.steps.InMemCuboidMapper.cleanup(InMemCuboidMapper.java:143) ... 8 more Caused by: java.lang.RuntimeException: java.io.IOException: java.io.IOException: java.lang.IllegalStateException at org.apache.kylin.cube.inmemcubing.AbstractInMemCubeBuilder$1.run(AbstractInMemCubeBuilder.java:84) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Caused by: java.io.IOException: java.io.IOException: java.lang.IllegalStateException at org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder$BuildOnce.build(DoggedCubeBuilder.java:128) at org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder.build(DoggedCubeBuilder.java:75) at org.apache.kylin.cube.inmemcubing.AbstractInMemCubeBuilder$1.run(AbstractInMemCubeBuilder.java:82) ... 5 more Caused by: java.io.IOException: java.lang.IllegalStateException at org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder$BuildOnce.abort(DoggedCubeBuilder.java:196) at org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder$BuildOnce.checkException(DoggedCubeBuilder.java:169) at org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder$BuildOnce.build(DoggedCubeBuilder.java:101) ... 7 more Caused by: java.lang.IllegalStateException at com.google.common.base.Preconditions.checkState(Preconditions.java:129) at org.apache.kylin.common.util.MemoryBudgetController.<init>(MemoryBudgetController.java:73) at org.apache.kylin.cube.inmemcubing.InMemCubeBuilder.makeMemoryBudget(InMemCubeBuilder.java:324) at org.apache.kylin.cube.inmemcubing.InMemCubeBuilder.build(InMemCubeBuilder.java:174) at org.apache.kylin.cube.inmemcubing.InMemCubeBuilder.build(InMemCubeBuilder.java:137) at org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder$SplitThread.run(DoggedCubeBuilder.java:284)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)