You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by "Billy Liu (JIRA)" <ji...@apache.org> on 2018/01/24 16:00:00 UTC
[jira] [Resolved] (KYLIN-2634) HBaseResourceStore should throw
clearer error
[ https://issues.apache.org/jira/browse/KYLIN-2634?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Billy Liu resolved KYLIN-2634.
------------------------------
Resolution: Fixed
Assignee: liyang (was: Dong Li)
Fix Version/s: v2.1.0
Commit: https://github.com/apache/kylin/commit/eafbe73
> HBaseResourceStore should throw clearer error
> ---------------------------------------------
>
> Key: KYLIN-2634
> URL: https://issues.apache.org/jira/browse/KYLIN-2634
> Project: Kylin
> Issue Type: Bug
> Components: Job Engine
> Affects Versions: v2.0.0
> Environment: CentOS release 6.8 (Final) x64
> CDH 5.9
> Reporter: flashput
> Assignee: liyang
> Priority: Major
> Fix For: v2.1.0
>
> Attachments: kylin.log, kylin.properties, kylin_hive_conf.xml, kylin_job_conf.xml
>
>
> bq.
> +----------+----------+----------+----------+----------+----------+
> | ip | u_domain | u_page | r_domain | r_page | agent |
> +----------+----------+----------+----------+----------+----------+
> | 20000519 | 20000012 | 20118849 | 20000000 | 20000000 | 20000022 |
> | 20000113 | 20000012 | 20118850 | 20000000 | 20000000 | 20000022 |
> | 20000247 | 20000012 | 20118851 | 20000000 | 20000000 | 20000022 |
> | 20000325 | 20000012 | 20118852 | 20000000 | 20000000 | 20000022 |
> | 20000247 | 20000012 | 20118853 | 20000000 | 20000000 | 20000022 |
> +----------+----------+----------+----------+----------+----------+
> +----------+
> | count(*) |
> +----------+
> | 25452592 |
> +----------+
> Model description:
> {
> "uuid": "c39058c4-3e9d-4c0c-a908-c8efef41cc91",
> "last_modified": 1495117591531,
> "version": "2.0.0",
> "name": "LOG_PV",
> "owner": "ADMIN",
> "description": "",
> "fact_table": "DEFAULT.LOG_DATA_170416",
> "lookups": [],
> "dimensions": [
> {
> "table": "LOG_DATA_170416",
> "columns": [
> "U_DOMAIN",
> "U_PAGE",
> "R_DOMAIN",
> "R_PAGE",
> "AGENT",
> "IP"
> ]
> }
> ],
> "metrics": [
> "LOG_DATA_170416.LOAD_TIME",
> "LOG_DATA_170416.ARTICLE_CONTENT_HEIGHT"
> ],
> "filter_condition": "",
> "partition_desc": {
> "partition_date_column": null,
> "partition_time_column": null,
> "partition_date_start": 0,
> "partition_date_format": "yyyyMMdd",
> "partition_time_format": "HH:mm:ss",
> "partition_type": "APPEND",
> "partition_condition_builder": "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
> },
> "capacity": "MEDIUM"
> }
> Cube description json:
> {
> "uuid": "b475f98a-1ec2-45ad-a2eb-90217aa83d9b",
> "last_modified": 1495117617084,
> "version": "2.0.0",
> "name": "cc",
> "model_name": "LOG_PV",
> "description": "",
> "null_string": null,
> "dimensions": [
> {
> "name": "IP",
> "table": "LOG_DATA_170416",
> "column": "IP",
> "derived": null
> },
> {
> "name": "U_DOMAIN",
> "table": "LOG_DATA_170416",
> "column": "U_DOMAIN",
> "derived": null
> },
> {
> "name": "U_PAGE",
> "table": "LOG_DATA_170416",
> "column": "U_PAGE",
> "derived": null
> },
> {
> "name": "R_DOMAIN",
> "table": "LOG_DATA_170416",
> "column": "R_DOMAIN",
> "derived": null
> },
> {
> "name": "R_PAGE",
> "table": "LOG_DATA_170416",
> "column": "R_PAGE",
> "derived": null
> },
> {
> "name": "AGENT",
> "table": "LOG_DATA_170416",
> "column": "AGENT",
> "derived": null
> }
> ],
> "measures": [
> {
> "name": "_COUNT_",
> "function": {
> "expression": "COUNT",
> "parameter": {
> "type": "constant",
> "value": "1"
> },
> "returntype": "bigint"
> }
> }
> ],
> "dictionaries": [],
> "rowkey": {
> "rowkey_columns": [
> {
> "column": "LOG_DATA_170416.IP",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.U_DOMAIN",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.U_PAGE",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.R_DOMAIN",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.R_PAGE",
> "encoding": "dict",
> "isShardBy": false
> },
> {
> "column": "LOG_DATA_170416.AGENT",
> "encoding": "dict",
> "isShardBy": false
> }
> ]
> },
> "hbase_mapping": {
> "column_family": [
> {
> "name": "F1",
> "columns": [
> {
> "qualifier": "M",
> "measure_refs": [
> "_COUNT_"
> ]
> }
> ]
> }
> ]
> },
> "aggregation_groups": [
> {
> "includes": [
> "LOG_DATA_170416.IP",
> "LOG_DATA_170416.U_DOMAIN",
> "LOG_DATA_170416.U_PAGE",
> "LOG_DATA_170416.R_DOMAIN",
> "LOG_DATA_170416.R_PAGE",
> "LOG_DATA_170416.AGENT"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [],
> "joint_dims": []
> }
> }
> ],
> "signature": "ALTH7M0wbIA5eA96sGbvCg==",
> "notify_list": [],
> "status_need_notify": [
> "ERROR",
> "DISCARDED",
> "SUCCEED"
> ],
> "partition_date_start": 0,
> "partition_date_end": 3153600000000,
> "auto_merge_time_ranges": [
> 604800000,
> 2419200000
> ],
> "retention_range": 0,
> "engine_type": 4,
> "storage_type": 2,
> "override_kylin_properties": {}
> }
> File not found Exception occured when doing Cube building .
> build stops at #4 Step Name: Build Dimension Dictionary
> Job error log:
> java.io.FileNotFoundException: File does not exist: /kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
> at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
> at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
> at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
> at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
> at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
> at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
> at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
> at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
> at org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1280)
> at org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1265)
> at org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1253)
> at org.apache.hadoop.hdfs.DFSInputStream.fetchLocatedBlocksAndGetLastBlockLength(DFSInputStream.java:305)
> at org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:271)
> at org.apache.hadoop.hdfs.DFSInputStream.<init>(DFSInputStream.java:263)
> at org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1584)
> at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:309)
> at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:305)
> at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> at org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:305)
> at org.apache.hadoop.fs.FileSystem.open(FileSystem.java:780)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.getInputStream(HBaseResourceStore.java:207)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.access$000(HBaseResourceStore.java:58)
> at org.apache.kylin.storage.hbase.HBaseResourceStore$2.visit(HBaseResourceStore.java:173)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.visitFolder(HBaseResourceStore.java:143)
> at org.apache.kylin.storage.hbase.HBaseResourceStore.getAllResourcesImpl(HBaseResourceStore.java:168)
> at org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:182)
> at org.apache.kylin.common.persistence.ResourceStore.getAllResources(ResourceStore.java:175)
> at org.apache.kylin.dict.DictionaryManager.checkDupByInfo(DictionaryManager.java:374)
> at org.apache.kylin.dict.DictionaryManager.saveDictionary(DictionaryManager.java:318)
> at org.apache.kylin.cube.CubeManager.saveDictionary(CubeManager.java:234)
> at org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:68)
> at org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:54)
> at org.apache.kylin.engine.mr.steps.CreateDictionaryJob.run(CreateDictionaryJob.java:66)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
> at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
> at org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:63)
> at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
> at org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:64)
> at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:124)
> at org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:142)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.hadoop.ipc.RemoteException(java.io.FileNotFoundException): File does not exist: /kylin/kylin_metadata/resources/dict/DEFAULT.LOG_DATA_170416/IP/56abfe1f-1fdf-4bec-baec-43721e693c32.dict
> at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:66)
> at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1963)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1904)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1884)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1856)
> at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:568)
> at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:89)
> at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:363)
> at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2086)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2082)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2080)
> at org.apache.hadoop.ipc.Client.call(Client.java:1471)
> at org.apache.hadoop.ipc.Client.call(Client.java:1408)
> at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230)
> at com.sun.proxy.$Proxy30.getBlockLocations(Unknown Source)
> at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:256)
> at sun.reflect.GeneratedMethodAccessor68.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:256)
> at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)
> at com.sun.proxy.$Proxy31.getBlockLocations(Unknown Source)
> at org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1278)
> ... 34 more
> result code:2
> server configuration file added as attachment.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)