You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by "Xingxing Di (JIRA)" <ji...@apache.org> on 2016/04/07 09:45:25 UTC
[jira] [Updated] (KYLIN-1568) Got java.nio.BufferOverflowException
when Build Base Cuboid Data
[ https://issues.apache.org/jira/browse/KYLIN-1568?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Xingxing Di updated KYLIN-1568:
-------------------------------
Description:
I looked into the code , found this :
valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
It looks like the ROWVALUE_BUFFER_SIZE is smaller than the value bytes length.
The exeption is :
2016-04-07 10:18:57,311 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : java.nio.BufferOverflowException at java.nio.HeapByteBuffer.put(HeapByteBuffer.java:200) at org.apache.kylin.measure.bitmap.BitmapCounter.writeRegisters(BitmapCounter.java:85) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:40) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:30) at org.apache.kylin.measure.MeasureCodec.encode(MeasureCodec.java:89) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:105) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:43) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)
The cube info :
{
"uuid": "6990333d-ef59-44e3-b46a-7e571d002698",
"version": "1.5.0",
"name": "flw_test2_cube_1",
"description": "",
"dimensions": [
{
"name": "TMP.TMP_FLW_MAIN_2.SITE_CLASS",
"table": "TMP.TMP_FLW_MAIN_2",
"column": "SITE_CLASS",
"derived": null
},
{
"name": "TMP.TMP_FLW_MAIN_2.DEVICE_ID",
"table": "TMP.TMP_FLW_MAIN_2",
"column": "DEVICE_ID",
"derived": null
},
{
"name": "TMP.TMP_FLW_MAIN_2.REF_DOMAIN",
"table": "TMP.TMP_FLW_MAIN_2",
"column": "REF_DOMAIN",
"derived": null
},
{
"name": "TMP.TMP_FLW_MAIN_2.NEW_FLAG",
"table": "TMP.TMP_FLW_MAIN_2",
"column": "NEW_FLAG",
"derived": null
},
{
"name": "TMP.TMP_FLW_MAIN_2.DEALER_ID",
"table": "TMP.TMP_FLW_MAIN_2",
"column": "DEALER_ID",
"derived": null
},
{
"name": "TMP.TMP_FLW_MAIN_2.KEY_WORD",
"table": "TMP.TMP_FLW_MAIN_2",
"column": "KEY_WORD",
"derived": null
},
{
"name": "TMP.OLAP_DIM_CITY_ALL_DERIVED",
"table": "TMP.OLAP_DIM_CITY_ALL",
"column": null,
"derived": [
"ID",
"CITY_ID",
"CITY_NAME",
"PROVINCE_ID",
"PROVINCE_NAME"
]
},
{
"name": "TMP.OLAP_DIM_SITE_ALL_DERIVED",
"table": "TMP.OLAP_DIM_SITE_ALL",
"column": null,
"derived": [
"ID",
"SITE_ID",
"SITE_NAME",
"CATEGORY_ID",
"CATEGORY_NAME",
"SUB_CATEGORY_ID",
"SUB_CATEGORY_NAME"
]
},
{
"name": "OLAP.OLAP_DIM_DAY_DERIVED",
"table": "OLAP.OLAP_DIM_DAY",
"column": null,
"derived": [
"DIM_DAY_ID",
"DAY_DATE",
"WEEKDAY_ID",
"DIM_MONTH_ID"
]
},
{
"name": "TMP.OLAP_DIM_SPEC_ALL_DERIVED",
"table": "TMP.OLAP_DIM_SPEC_ALL",
"column": null,
"derived": [
"ID",
"SPEC_ID",
"SPEC_NAME",
"SERIES_ID",
"SERIES_NAME",
"BRAND_ID",
"BRAND_NAME",
"LEVEL_ID",
"LEVEL_NAME",
"FACTORY_ID",
"FACTORY_NAME"
]
}
],
"measures": [
{
"name": "_COUNT_",
"function": {
"expression": "COUNT",
"parameter": {
"type": "constant",
"value": "1",
"next_parameter": null
},
"returntype": "bigint"
},
"dependent_measure_ref": null
},
{
"name": "PV",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "PV",
"next_parameter": null
},
"returntype": "bigint"
},
"dependent_measure_ref": null
},
{
"name": "USER_ID",
"function": {
"expression": "COUNT_DISTINCT",
"parameter": {
"type": "column",
"value": "USER_ID",
"next_parameter": null
},
"returntype": "bitmap"
},
"dependent_measure_ref": null
}
],
"rowkey": {
"rowkey_columns": [
{
"column": "DT",
"encoding": "dict"
},
{
"column": "SPEC_ID",
"encoding": "dict"
},
{
"column": "SITE_ID",
"encoding": "dict"
},
{
"column": "DEALER_ID",
"encoding": "dict"
},
{
"column": "CITY_ID",
"encoding": "dict"
},
{
"column": "KEY_WORD",
"encoding": "dict"
},
{
"column": "REF_DOMAIN",
"encoding": "dict"
},
{
"column": "SITE_CLASS",
"encoding": "dict"
},
{
"column": "DEVICE_ID",
"encoding": "dict"
},
{
"column": "NEW_FLAG",
"encoding": "dict"
}
]
},
"signature": "8O48Zjj6YujqabUmbVcg+A==",
"last_modified": 1459996931395,
"model_name": "flw_test2",
"null_string": null,
"hbase_mapping": {
"column_family": [
{
"name": "F1",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"_COUNT_",
"PV"
]
}
]
},
{
"name": "F2",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"USER_ID"
]
}
]
}
]
},
"aggregation_groups": [
{
"includes": [
"SPEC_ID",
"CITY_ID",
"DT",
"SITE_CLASS",
"SITE_ID"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [
"DT"
],
"joint_dims": [
[
"SITE_CLASS",
"CITY_ID",
"SPEC_ID"
]
]
}
},
{
"includes": [
"DT",
"SITE_CLASS",
"DEVICE_ID",
"REF_DOMAIN",
"SITE_ID"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [
"DT"
],
"joint_dims": [
[
"DEVICE_ID",
"REF_DOMAIN",
"SITE_ID"
]
]
}
},
{
"includes": [
"DT",
"NEW_FLAG",
"SPEC_ID",
"REF_DOMAIN",
"DEALER_ID"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [
"DT"
],
"joint_dims": []
}
},
{
"includes": [
"DT",
"KEY_WORD",
"SITE_ID",
"DEVICE_ID",
"REF_DOMAIN"
],
"select_rule": {
"hierarchy_dims": [],
"mandatory_dims": [
"DT"
],
"joint_dims": []
}
}
],
"notify_list": [],
"status_need_notify": [],
"partition_date_start": 1459814400000,
"partition_date_end": 3153600000000,
"auto_merge_time_ranges": [
604800000,
2419200000
],
"retention_range": 0,
"engine_type": 2,
"storage_type": 2,
"override_kylin_properties": {}
}
> Got java.nio.BufferOverflowException when Build Base Cuboid Data
> -----------------------------------------------------------------
>
> Key: KYLIN-1568
> URL: https://issues.apache.org/jira/browse/KYLIN-1568
> Project: Kylin
> Issue Type: Bug
> Components: Job Engine
> Affects Versions: v1.5.0
> Environment: hadoop2.4.1 , hive1.0 , hbase0.98
> Reporter: Xingxing Di
> Assignee: Dong Li
>
> I looked into the code , found this :
> valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
> It looks like the ROWVALUE_BUFFER_SIZE is smaller than the value bytes length.
> The exeption is :
> 2016-04-07 10:18:57,311 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : java.nio.BufferOverflowException at java.nio.HeapByteBuffer.put(HeapByteBuffer.java:200) at org.apache.kylin.measure.bitmap.BitmapCounter.writeRegisters(BitmapCounter.java:85) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:40) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:30) at org.apache.kylin.measure.MeasureCodec.encode(MeasureCodec.java:89) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:105) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:43) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)
> The cube info :
> {
> "uuid": "6990333d-ef59-44e3-b46a-7e571d002698",
> "version": "1.5.0",
> "name": "flw_test2_cube_1",
> "description": "",
> "dimensions": [
> {
> "name": "TMP.TMP_FLW_MAIN_2.SITE_CLASS",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "SITE_CLASS",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.DEVICE_ID",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "DEVICE_ID",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.REF_DOMAIN",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "REF_DOMAIN",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.NEW_FLAG",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "NEW_FLAG",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.DEALER_ID",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "DEALER_ID",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.KEY_WORD",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "KEY_WORD",
> "derived": null
> },
> {
> "name": "TMP.OLAP_DIM_CITY_ALL_DERIVED",
> "table": "TMP.OLAP_DIM_CITY_ALL",
> "column": null,
> "derived": [
> "ID",
> "CITY_ID",
> "CITY_NAME",
> "PROVINCE_ID",
> "PROVINCE_NAME"
> ]
> },
> {
> "name": "TMP.OLAP_DIM_SITE_ALL_DERIVED",
> "table": "TMP.OLAP_DIM_SITE_ALL",
> "column": null,
> "derived": [
> "ID",
> "SITE_ID",
> "SITE_NAME",
> "CATEGORY_ID",
> "CATEGORY_NAME",
> "SUB_CATEGORY_ID",
> "SUB_CATEGORY_NAME"
> ]
> },
> {
> "name": "OLAP.OLAP_DIM_DAY_DERIVED",
> "table": "OLAP.OLAP_DIM_DAY",
> "column": null,
> "derived": [
> "DIM_DAY_ID",
> "DAY_DATE",
> "WEEKDAY_ID",
> "DIM_MONTH_ID"
> ]
> },
> {
> "name": "TMP.OLAP_DIM_SPEC_ALL_DERIVED",
> "table": "TMP.OLAP_DIM_SPEC_ALL",
> "column": null,
> "derived": [
> "ID",
> "SPEC_ID",
> "SPEC_NAME",
> "SERIES_ID",
> "SERIES_NAME",
> "BRAND_ID",
> "BRAND_NAME",
> "LEVEL_ID",
> "LEVEL_NAME",
> "FACTORY_ID",
> "FACTORY_NAME"
> ]
> }
> ],
> "measures": [
> {
> "name": "_COUNT_",
> "function": {
> "expression": "COUNT",
> "parameter": {
> "type": "constant",
> "value": "1",
> "next_parameter": null
> },
> "returntype": "bigint"
> },
> "dependent_measure_ref": null
> },
> {
> "name": "PV",
> "function": {
> "expression": "SUM",
> "parameter": {
> "type": "column",
> "value": "PV",
> "next_parameter": null
> },
> "returntype": "bigint"
> },
> "dependent_measure_ref": null
> },
> {
> "name": "USER_ID",
> "function": {
> "expression": "COUNT_DISTINCT",
> "parameter": {
> "type": "column",
> "value": "USER_ID",
> "next_parameter": null
> },
> "returntype": "bitmap"
> },
> "dependent_measure_ref": null
> }
> ],
> "rowkey": {
> "rowkey_columns": [
> {
> "column": "DT",
> "encoding": "dict"
> },
> {
> "column": "SPEC_ID",
> "encoding": "dict"
> },
> {
> "column": "SITE_ID",
> "encoding": "dict"
> },
> {
> "column": "DEALER_ID",
> "encoding": "dict"
> },
> {
> "column": "CITY_ID",
> "encoding": "dict"
> },
> {
> "column": "KEY_WORD",
> "encoding": "dict"
> },
> {
> "column": "REF_DOMAIN",
> "encoding": "dict"
> },
> {
> "column": "SITE_CLASS",
> "encoding": "dict"
> },
> {
> "column": "DEVICE_ID",
> "encoding": "dict"
> },
> {
> "column": "NEW_FLAG",
> "encoding": "dict"
> }
> ]
> },
> "signature": "8O48Zjj6YujqabUmbVcg+A==",
> "last_modified": 1459996931395,
> "model_name": "flw_test2",
> "null_string": null,
> "hbase_mapping": {
> "column_family": [
> {
> "name": "F1",
> "columns": [
> {
> "qualifier": "M",
> "measure_refs": [
> "_COUNT_",
> "PV"
> ]
> }
> ]
> },
> {
> "name": "F2",
> "columns": [
> {
> "qualifier": "M",
> "measure_refs": [
> "USER_ID"
> ]
> }
> ]
> }
> ]
> },
> "aggregation_groups": [
> {
> "includes": [
> "SPEC_ID",
> "CITY_ID",
> "DT",
> "SITE_CLASS",
> "SITE_ID"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": [
> [
> "SITE_CLASS",
> "CITY_ID",
> "SPEC_ID"
> ]
> ]
> }
> },
> {
> "includes": [
> "DT",
> "SITE_CLASS",
> "DEVICE_ID",
> "REF_DOMAIN",
> "SITE_ID"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": [
> [
> "DEVICE_ID",
> "REF_DOMAIN",
> "SITE_ID"
> ]
> ]
> }
> },
> {
> "includes": [
> "DT",
> "NEW_FLAG",
> "SPEC_ID",
> "REF_DOMAIN",
> "DEALER_ID"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": []
> }
> },
> {
> "includes": [
> "DT",
> "KEY_WORD",
> "SITE_ID",
> "DEVICE_ID",
> "REF_DOMAIN"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": []
> }
> }
> ],
> "notify_list": [],
> "status_need_notify": [],
> "partition_date_start": 1459814400000,
> "partition_date_end": 3153600000000,
> "auto_merge_time_ranges": [
> 604800000,
> 2419200000
> ],
> "retention_range": 0,
> "engine_type": 2,
> "storage_type": 2,
> "override_kylin_properties": {}
> }
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)