You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by "wangfei (Jira)" <ji...@apache.org> on 2020/07/08 09:32:00 UTC
[jira] [Updated] (KYLIN-4624) kylin3.1.0使用flink1.10作为engine报错问题
[ https://issues.apache.org/jira/browse/KYLIN-4624?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
wangfei updated KYLIN-4624:
---------------------------
Description:
{color:#ff0000}在执行到如下步骤出错：{color}
#6 Step Name: Build Cube with Flink
Duration: 0.02 mins Waiting: 0 seconds
{color:#ff0000}报错：{color}
java.lang.NoClassDefFoundError: Could not initialize class org.apache.kylin.engine.flink.FlinkOnYarnConfigMapping
{color:#ff0000}然后我去查找hadoop container的日志{color}
2020-07-08 08:40:34,395 ERROR [main] org.apache.kylin.common.persistence.ResourceStore: Error reading resource /cube/.order_cube_clone.json.crc
org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParseException: Invalid UTF-8 start byte 0x8f
at [Source: (DataInputStream); line: 1, column: 10]
2020-07-08 08:40:34,482 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.project.ProjectManager
2020-07-08 08:40:34,483 INFO [main] org.apache.kylin.metadata.project.ProjectManager: Initializing ProjectManager with metadata url kylin_metadata@ifile,path=/data/hadoop/nm-local-dir/usercache/root/appcache/application_1594187410626_0016/container_1594187410626_0016_01_000007/meta
2020-07-08 08:40:34,504 ERROR [main] org.apache.kylin.common.persistence.ResourceStore: Error reading resource /project/.test_flink.json.crc
org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'crc^@^@^@^B^@': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')
at [Source: (DataInputStream); line: 1, column: 11]
2020-07-08 08:40:34,512 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.cachesync.Broadcaster
2020-07-08 08:40:34,524 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.model.DataModelManager
2020-07-08 08:40:34,534 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.TableMetadataManager
2020-07-08 08:40:34,543 ERROR [main] org.apache.kylin.common.persistence.ResourceStore: Error reading resource /table/.TEST.CUSTOMER--test_flink.json.crc
org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParseException: Invalid UTF-8 middle byte 0xe4
at [Source: (DataInputStream); line: 1, column: 11]
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1840)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:712)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._reportInvalidOther(UTF8StreamJsonParser.java:3573)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._decodeCharForError(UTF8StreamJsonParser.java:3318)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._reportInvalidToken(UTF8StreamJsonParser.java:3544)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._handleUnexpectedValue(UTF8StreamJsonParser.java:2651)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._nextTokenNotInObject(UTF8StreamJsonParser.java:856)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:753)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4340)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4189)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3242)
at org.apache.kylin.common.util.JsonUtil.readValue(JsonUtil.java:71)
at org.apache.kylin.common.persistence.JsonSerializer.deserialize(JsonSerializer.java:46)
at org.apache.kylin.common.persistence.ContentReader.readContent(ContentReader.java:40)
at org.apache.kylin.common.persistence.ResourceStore$3.visit(ResourceStore.java:259)
at org.apache.kylin.common.persistence.FileResourceStore.visitFolderImpl(FileResourceStore.java:87)
at org.apache.kylin.common.persistence.ResourceStore.visitFolderInner(ResourceStore.java:766)
at org.apache.kylin.common.persistence.ResourceStore.visitFolderAndContent(ResourceStore.java:751)
at org.apache.kylin.common.persistence.ResourceStore.lambda$getAllResourcesMap$0(ResourceStore.java:255)
at org.apache.kylin.common.persistence.ExponentialBackoffRetry.doWithRetry(ExponentialBackoffRetry.java:52)
at org.apache.kylin.common.persistence.ResourceStore.getAllResourcesMap(ResourceStore.java:253)
at org.apache.kylin.metadata.cachesync.CachedCrudAssist.reloadAll(CachedCrudAssist.java:127)
at org.apache.kylin.metadata.TableMetadataManager.initSrcTable(TableMetadataManager.java:126)
at org.apache.kylin.metadata.TableMetadataManager.<init>(TableMetadataManager.java:98)
at org.apache.kylin.metadata.TableMetadataManager.newInstance(TableMetadataManager.java:73)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
was:
{color:#FF0000}在执行到如下步骤出错：{color}
#6 Step Name: Build Cube with Flink
Duration: 0.02 mins Waiting: 0 seconds
{color:#FF0000}报错：{color}
java.lang.NoClassDefFoundError: Could not initialize class org.apache.kylin.engine.flink.FlinkOnYarnConfigMapping
{color:#FF0000}然后我去查找hadoop container的日志{color}
2020-07-08 08:40:34,395 ERROR [main] org.apache.kylin.common.persistence.ResourceStore: Error reading resource /cube/.order_cube_clone.json.crc
org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParseException: Invalid UTF-8 start byte 0x8f
at [Source: (DataInputStream); line: 1, column: 10]
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1840)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:712)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:753)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4340)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4189)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3242)
at org.apache.kylin.common.util.JsonUtil.readValue(JsonUtil.java:71)
at org.apache.kylin.common.persistence.JsonSerializer.deserialize(JsonSerializer.java:46)
at org.apache.kylin.common.persistence.ContentReader.readContent(ContentReader.java:40)
at org.apache.kylin.common.persistence.ResourceStore$3.visit(ResourceStore.java:259)
at org.apache.kylin.common.persistence.FileResourceStore.visitFolderImpl(FileResourceStore.java:87)
at org.apache.kylin.common.persistence.ResourceStore.visitFolderInner(ResourceStore.java:766)
at org.apache.kylin.common.persistence.ResourceStore.visitFolderAndContent(ResourceStore.java:751)
at org.apache.kylin.common.persistence.ResourceStore.lambda$getAllResourcesMap$0(ResourceStore.java:255)
at org.apache.kylin.common.persistence.ExponentialBackoffRetry.doWithRetry(ExponentialBackoffRetry.java:52)
at org.apache.kylin.common.persistence.ResourceStore.getAllResourcesMap(ResourceStore.java:253)
at org.apache.kylin.metadata.cachesync.CachedCrudAssist.reloadAll(CachedCrudAssist.java:127)
at org.apache.kylin.cube.CubeDescManager.<init>(CubeDescManager.java:112)
at org.apache.kylin.cube.CubeDescManager.newInstance(CubeDescManager.java:75)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at org.apache.kylin.cube.CubeManager.newInstance(CubeManager.java:111)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:168)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
at [Source: (DataInputStream); line: 1, column: 11]
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1840)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:722)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:753)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4340)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4189)
at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3242)
at org.apache.kylin.common.util.JsonUtil.readValue(JsonUtil.java:71)
at org.apache.kylin.common.persistence.JsonSerializer.deserialize(JsonSerializer.java:46)
at org.apache.kylin.common.persistence.ContentReader.readContent(ContentReader.java:40)
at org.apache.kylin.common.persistence.ResourceStore$3.visit(ResourceStore.java:259)
at org.apache.kylin.common.persistence.FileResourceStore.visitFolderImpl(FileResourceStore.java:87)
at org.apache.kylin.common.persistence.ResourceStore.visitFolderInner(ResourceStore.java:766)
at org.apache.kylin.common.persistence.ResourceStore.visitFolderAndContent(ResourceStore.java:751)
at org.apache.kylin.common.persistence.ResourceStore.lambda$getAllResourcesMap$0(ResourceStore.java:255)
at org.apache.kylin.common.persistence.ExponentialBackoffRetry.doWithRetry(ExponentialBackoffRetry.java:52)
at org.apache.kylin.common.persistence.ResourceStore.getAllResourcesMap(ResourceStore.java:253)
at org.apache.kylin.metadata.cachesync.CachedCrudAssist.reloadAll(CachedCrudAssist.java:127)
at org.apache.kylin.metadata.project.ProjectManager.<init>(ProjectManager.java:99)
at org.apache.kylin.metadata.project.ProjectManager.newInstance(ProjectManager.java:68)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.kylin.common.KylinConfig.getManager(KylinConfig.java:481)
at org.apache.kylin.metadata.project.ProjectManager.getInstance(ProjectManager.java:63)
at org.apache.kylin.cube.model.CubeDesc.init(CubeDesc.java:677)
at org.apache.kylin.cube.CubeDescManager$1.initEntityAfterReload(CubeDescManager.java:102)
at org.apache.kylin.cube.CubeDescManager$1.initEntityAfterReload(CubeDescManager.java:95)
at org.apache.kylin.metadata.cachesync.CachedCrudAssist.reloadAll(CachedCrudAssist.java:144)
at org.apache.kylin.cube.CubeDescManager.<init>(CubeDescManager.java:112)
at org.apache.kylin.cube.CubeDescManager.newInstance(CubeDescManager.java:75)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.kylin.common.KylinConfig.getManager(KylinConfig.java:481)
at org.apache.kylin.cube.CubeDescManager.getInstance(CubeDescManager.java:67)
at org.apache.kylin.cube.CubeInstance.init(CubeInstance.java:147)
at org.apache.kylin.cube.CubeManager$1.initEntityAfterReload(CubeManager.java:143)
at org.apache.kylin.cube.CubeManager$1.initEntityAfterReload(CubeManager.java:140)
at org.apache.kylin.metadata.cachesync.CachedCrudAssist.reloadAll(CachedCrudAssist.java:144)
at org.apache.kylin.cube.CubeManager.<init>(CubeManager.java:154)
at org.apache.kylin.cube.CubeManager.newInstance(CubeManager.java:111)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.kylin.common.KylinConfig.getManager(KylinConfig.java:481)
at org.apache.kylin.cube.CubeManager.getInstance(CubeManager.java:106)
at org.apache.kylin.engine.mr.steps.FactDistinctColumnsReducer.doSetup(FactDistinctColumnsReducer.java:95)
at org.apache.kylin.engine.mr.KylinReducer.setup(KylinReducer.java:47)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:168)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
> kylin3.1.0使用flink1.10作为engine报错问题
> ---------------------------------
>
> Key: KYLIN-4624
> URL: https://issues.apache.org/jira/browse/KYLIN-4624
> Project: Kylin
> Issue Type: Bug
> Components: Job Engine
> Affects Versions: v3.1.0
> Environment: centos 7
> 使用的官网提供的docker镜像，然后我给镜像加了flink
> flink1.10
> kylin3.1.0
> Reporter: wangfei
> Priority: Major
> Fix For: v3.1.0
>
>
> {color:#ff0000}在执行到如下步骤出错：{color}
> #6 Step Name: Build Cube with Flink
> Duration: 0.02 mins Waiting: 0 seconds
> {color:#ff0000}报错：{color}
> java.lang.NoClassDefFoundError: Could not initialize class org.apache.kylin.engine.flink.FlinkOnYarnConfigMapping
> {color:#ff0000}然后我去查找hadoop container的日志{color}
> 2020-07-08 08:40:34,395 ERROR [main] org.apache.kylin.common.persistence.ResourceStore: Error reading resource /cube/.order_cube_clone.json.crc
> org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParseException: Invalid UTF-8 start byte 0x8f
> at [Source: (DataInputStream); line: 1, column: 10]
>
> 2020-07-08 08:40:34,482 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.project.ProjectManager
> 2020-07-08 08:40:34,483 INFO [main] org.apache.kylin.metadata.project.ProjectManager: Initializing ProjectManager with metadata url kylin_metadata@ifile,path=/data/hadoop/nm-local-dir/usercache/root/appcache/application_1594187410626_0016/container_1594187410626_0016_01_000007/meta
> 2020-07-08 08:40:34,504 ERROR [main] org.apache.kylin.common.persistence.ResourceStore: Error reading resource /project/.test_flink.json.crc
> org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'crc^@^@^@^B^@': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')
> at [Source: (DataInputStream); line: 1, column: 11]
>
>
> 2020-07-08 08:40:34,512 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.cachesync.Broadcaster
> 2020-07-08 08:40:34,524 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.model.DataModelManager
> 2020-07-08 08:40:34,534 INFO [main] org.apache.kylin.common.KylinConfig: Creating new manager instance of class org.apache.kylin.metadata.TableMetadataManager
> 2020-07-08 08:40:34,543 ERROR [main] org.apache.kylin.common.persistence.ResourceStore: Error reading resource /table/.TEST.CUSTOMER--test_flink.json.crc
> org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParseException: Invalid UTF-8 middle byte 0xe4
> at [Source: (DataInputStream); line: 1, column: 11]
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1840)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:712)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._reportInvalidOther(UTF8StreamJsonParser.java:3573)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._decodeCharForError(UTF8StreamJsonParser.java:3318)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._reportInvalidToken(UTF8StreamJsonParser.java:3544)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._handleUnexpectedValue(UTF8StreamJsonParser.java:2651)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser._nextTokenNotInObject(UTF8StreamJsonParser.java:856)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:753)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4340)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4189)
> at org.apache.kylin.job.shaded.com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3242)
> at org.apache.kylin.common.util.JsonUtil.readValue(JsonUtil.java:71)
> at org.apache.kylin.common.persistence.JsonSerializer.deserialize(JsonSerializer.java:46)
> at org.apache.kylin.common.persistence.ContentReader.readContent(ContentReader.java:40)
> at org.apache.kylin.common.persistence.ResourceStore$3.visit(ResourceStore.java:259)
> at org.apache.kylin.common.persistence.FileResourceStore.visitFolderImpl(FileResourceStore.java:87)
> at org.apache.kylin.common.persistence.ResourceStore.visitFolderInner(ResourceStore.java:766)
> at org.apache.kylin.common.persistence.ResourceStore.visitFolderAndContent(ResourceStore.java:751)
> at org.apache.kylin.common.persistence.ResourceStore.lambda$getAllResourcesMap$0(ResourceStore.java:255)
> at org.apache.kylin.common.persistence.ExponentialBackoffRetry.doWithRetry(ExponentialBackoffRetry.java:52)
> at org.apache.kylin.common.persistence.ResourceStore.getAllResourcesMap(ResourceStore.java:253)
> at org.apache.kylin.metadata.cachesync.CachedCrudAssist.reloadAll(CachedCrudAssist.java:127)
> at org.apache.kylin.metadata.TableMetadataManager.initSrcTable(TableMetadataManager.java:126)
> at org.apache.kylin.metadata.TableMetadataManager.<init>(TableMetadataManager.java:98)
> at org.apache.kylin.metadata.TableMetadataManager.newInstance(TableMetadataManager.java:73)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)