You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@kylin.apache.org by Liufeng <us...@126.com> on 2020/05/12 02:37:19 UTC
streaming cube full build error
您好：
我创建了一个streaming cube，一开始的时候，一切都是好的，但是每次FULL BUILD的时候，都会出错：
下面是错误消息：
java.io.FileNotFoundException: File hdfs://xxxx:8020/kylin/kylin_metadata/stream/EQUIP_COURSE_SUM_CUBE/FULL_BUILD does not exist.
at org.apache.hadoop.hdfs.DistributedFileSystem.listStatusInternal(DistributedFileSystem.java:748)
at org.apache.hadoop.hdfs.DistributedFileSystem.access$600(DistributedFileSystem.java:114)
at org.apache.hadoop.hdfs.DistributedFileSystem$16.doCall(DistributedFileSystem.java:809)
at org.apache.hadoop.hdfs.DistributedFileSystem$16.doCall(DistributedFileSystem.java:805)
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at org.apache.hadoop.hdfs.DistributedFileSystem.listStatus(DistributedFileSystem.java:805)
at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1566)
at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1609)
at org.apache.kylin.engine.mr.streaming.ColumnarSplitInputFormat.listStatus(ColumnarSplitInputFormat.java:60)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:387)
at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:305)
at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:322)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:200)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1307)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1304)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1304)
at org.apache.kylin.engine.mr.common.AbstractHadoopJob.waitForCompletion(AbstractHadoopJob.java:192)
at org.apache.kylin.engine.mr.streaming.MergeDictJob.run(MergeDictJob.java:95)
at org.apache.kylin.engine.mr.common.MapReduceExecutable.doWork(MapReduceExecutable.java:144)
at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:179)
at org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:71)
at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:179)
at org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
cube的定义如下：
{
"uuid": "e06feff9-4941-a76c-0eb0-938306279367",
"last_modified": 1584447176830,
"version": "3.0.0.20500",
"name": "EQUIP_COURSE_SUM_CUBE",
"is_draft": false,
"model_name": "EQUIP_COURSE_SUM_MODEL",
"description": "cube",
"null_string": null,
"dimensions": [
{
"name": "RECVTIME",
"table": "EQUIP_STATE_HISTORY",
"column": "RECVTIME",
"derived": null
},
{
"name": "EQUIPTYPE",
"table": "EQUIP_STATE_HISTORY",
"column": "EQUIPTYPE",
"derived": null
},
{
"name": "OSTYPE",
"table": "EQUIP_STATE_HISTORY",
"column": "OSTYPE",
"derived": null
},
{
"name": "SCHOOLID",
"table": "EQUIP_STATE_HISTORY",
"column": "SCHOOLID",
"derived": null
},
{
"name": "GRADEID",
"table": "EQUIP_STATE_HISTORY",
"column": "GRADEID",
"derived": null
},
{
"name": "CLASSID",
"table": "EQUIP_STATE_HISTORY",
"column": "CLASSID",
"derived": null
},
{
"name": "COURSEDICTID",
"table": "EQUIP_STATE_HISTORY",
"column": "COURSEDICTID",
"derived": null
},
{
"name": "TEACHERID",
"table": "EQUIP_STATE_HISTORY",
"column": "TEACHERID",
"derived": null
},
{
"name": "TEACHERAGE",
"table": "EQUIP_STATE_HISTORY",
"column": "TEACHERAGE",
"derived": null
},
{
"name": "OPENSTATUS",
"table": "EQUIP_STATE_HISTORY",
"column": "OPENSTATUS",
"derived": null
},
{
"name": "PROVINCE",
"table": "EQUIP_STATE_HISTORY",
"column": "PROVINCE",
"derived": null
},
{
"name": "CITY",
"table": "EQUIP_STATE_HISTORY",
"column": "CITY",
"derived": null
},
{
"name": "DISTRICT",
"table": "EQUIP_STATE_HISTORY",
"column": "DISTRICT",
"derived": null
},
{
"name": "YEAR_START",
"table": "EQUIP_STATE_HISTORY",
"column": "YEAR_START",
"derived": null
},
{
"name": "DAY_START",
"table": "EQUIP_STATE_HISTORY",
"column": "DAY_START",
"derived": null
},
{
"name": "HOUR_START",
"table": "EQUIP_STATE_HISTORY",
"column": "HOUR_START",
"derived": null
},
{
"name": "EQUIPID",
"table": "EQUIP_STATE_HISTORY",
"column": "EQUIPID",
"derived": null
}
],
"measures": [
{
"name": "_COUNT_",
"function": {
"expression": "COUNT",
"parameter": {
"type": "constant",
"value": "1"
},
"returntype": "bigint"
}
},
{
"name": "SUM_OPENTIMEDELTA",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "EQUIP_STATE_HISTORY.OPENTIMEDELTA"
},
"returntype": "bigint"
}
},
{
"name": "COUNT_DISTINCT_LESSON_NUM",
"function": {
"expression": "COUNT_DISTINCT",
"parameter": {
"type": "column",
"value": "EQUIP_STATE_HISTORY.LESSONNUM"
},
"returntype": "hllc(10)"
}
},
{
"name": "COUNT_DISTINCT_EQUIPID",
"function": {
"expression": "COUNT_DISTINCT",
"parameter": {
"type": "column",
"value": "EQUIP_STATE_HISTORY.EQUIPID"
},
"returntype": "hllc(10)"
}
}
],
"dictionaries": [],
"rowkey": {
"rowkey_columns": [
{
"column": "EQUIP_STATE_HISTORY.RECVTIME",
"encoding": "time",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.EQUIPTYPE",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.OSTYPE",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.SCHOOLID",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.COURSEDICTID",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.OPENSTATUS",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.PROVINCE",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.CITY",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.DISTRICT",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.YEAR_START",
"encoding": "date",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.DAY_START",
"encoding": "date",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.HOUR_START",
"encoding": "time",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.TEACHERID",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.TEACHERAGE",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.EQUIPID",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.GRADEID",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
},
{
"column": "EQUIP_STATE_HISTORY.CLASSID",
"encoding": "dict",
"encoding_version": 1,
"isShardBy": false
}
]
},
"hbase_mapping": {
"column_family": [
{
"name": "F1",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"_COUNT_",
"SUM_OPENTIMEDELTA"
]
}
]
},
{
"name": "F2",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"COUNT_DISTINCT_LESSON_NUM",
"COUNT_DISTINCT_EQUIPID"
]
}
]
}
]
},
"aggregation_groups": [
{
"includes": [
"EQUIP_STATE_HISTORY.RECVTIME",
"EQUIP_STATE_HISTORY.EQUIPTYPE",
"EQUIP_STATE_HISTORY.OSTYPE",
"EQUIP_STATE_HISTORY.SCHOOLID",
"EQUIP_STATE_HISTORY.COURSEDICTID",
"EQUIP_STATE_HISTORY.OPENSTATUS",
"EQUIP_STATE_HISTORY.PROVINCE",
"EQUIP_STATE_HISTORY.CITY",
"EQUIP_STATE_HISTORY.DISTRICT",
"EQUIP_STATE_HISTORY.YEAR_START",
"EQUIP_STATE_HISTORY.DAY_START",
"EQUIP_STATE_HISTORY.HOUR_START"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [],
"joint_dims": []
}
}
],
"signature": "UKjDGg3W2MyDzWE2V1UY0A==",
"notify_list": [],
"status_need_notify": [
"ERROR",
"DISCARDED",
"SUCCEED"
],
"partition_date_start": 0,
"partition_date_end": 3153600000000,
"auto_merge_time_ranges": [
604800000,
2419200000
],
"volatile_range": 0,
"retention_range": 0,
"engine_type": 2,
"storage_type": 3,
"override_kylin_properties": {
"kylin.stream.cube.window": "3600",
"kylin.stream.cube.duration": "3600",
"kylin.stream.index.checkpoint.intervals": "300",
"kylin.cube.algorithm": "INMEM",
"kylin.stream.segment.retention.policy": "fullBuild"
},
"cuboid_black_list": [],
"parent_forward": 3,
"mandatory_dimension_set_list": [],
"snapshot_table_desc_list": []
}