You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kylin.apache.org by "Shaofeng SHI (JIRA)" <ji...@apache.org> on 2016/07/28 06:49:22 UTC
[jira] [Closed] (KYLIN-1568) Calculate row value buffer size
instead of hard coded ROWVALUE_BUFFER_SIZE
[ https://issues.apache.org/jira/browse/KYLIN-1568?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Shaofeng SHI closed KYLIN-1568.
-------------------------------
Resolved in release 1.5.3 (2-16-07-28)
> Calculate row value buffer size instead of hard coded ROWVALUE_BUFFER_SIZE
> --------------------------------------------------------------------------
>
> Key: KYLIN-1568
> URL: https://issues.apache.org/jira/browse/KYLIN-1568
> Project: Kylin
> Issue Type: Bug
> Components: Job Engine
> Affects Versions: v1.5.0
> Environment: hadoop2.4.1 , hive1.0 , hbase0.98
> Reporter: Xingxing Di
> Assignee: liyang
> Labels: newbie
> Fix For: v1.5.3
>
>
> I looked into the code , found this :
> valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
> It looks like the ROWVALUE_BUFFER_SIZE is smaller than the value bytes length.
> The exeption is :
> 2016-04-07 10:18:57,311 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : java.nio.BufferOverflowException at java.nio.HeapByteBuffer.put(HeapByteBuffer.java:200) at org.apache.kylin.measure.bitmap.BitmapCounter.writeRegisters(BitmapCounter.java:85) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:40) at org.apache.kylin.measure.bitmap.BitmapSerializer.serialize(BitmapSerializer.java:30) at org.apache.kylin.measure.MeasureCodec.encode(MeasureCodec.java:89) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:105) at org.apache.kylin.engine.mr.steps.CuboidReducer.reduce(CuboidReducer.java:43) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389) at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:167) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1556) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:162)
> The cube info :
> {
> "uuid": "6990333d-ef59-44e3-b46a-7e571d002698",
> "version": "1.5.0",
> "name": "flw_test2_cube_1",
> "description": "",
> "dimensions": [
> {
> "name": "TMP.TMP_FLW_MAIN_2.SITE_CLASS",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "SITE_CLASS",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.DEVICE_ID",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "DEVICE_ID",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.REF_DOMAIN",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "REF_DOMAIN",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.NEW_FLAG",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "NEW_FLAG",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.DEALER_ID",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "DEALER_ID",
> "derived": null
> },
> {
> "name": "TMP.TMP_FLW_MAIN_2.KEY_WORD",
> "table": "TMP.TMP_FLW_MAIN_2",
> "column": "KEY_WORD",
> "derived": null
> },
> {
> "name": "TMP.OLAP_DIM_CITY_ALL_DERIVED",
> "table": "TMP.OLAP_DIM_CITY_ALL",
> "column": null,
> "derived": [
> "ID",
> "CITY_ID",
> "CITY_NAME",
> "PROVINCE_ID",
> "PROVINCE_NAME"
> ]
> },
> {
> "name": "TMP.OLAP_DIM_SITE_ALL_DERIVED",
> "table": "TMP.OLAP_DIM_SITE_ALL",
> "column": null,
> "derived": [
> "ID",
> "SITE_ID",
> "SITE_NAME",
> "CATEGORY_ID",
> "CATEGORY_NAME",
> "SUB_CATEGORY_ID",
> "SUB_CATEGORY_NAME"
> ]
> },
> {
> "name": "OLAP.OLAP_DIM_DAY_DERIVED",
> "table": "OLAP.OLAP_DIM_DAY",
> "column": null,
> "derived": [
> "DIM_DAY_ID",
> "DAY_DATE",
> "WEEKDAY_ID",
> "DIM_MONTH_ID"
> ]
> },
> {
> "name": "TMP.OLAP_DIM_SPEC_ALL_DERIVED",
> "table": "TMP.OLAP_DIM_SPEC_ALL",
> "column": null,
> "derived": [
> "ID",
> "SPEC_ID",
> "SPEC_NAME",
> "SERIES_ID",
> "SERIES_NAME",
> "BRAND_ID",
> "BRAND_NAME",
> "LEVEL_ID",
> "LEVEL_NAME",
> "FACTORY_ID",
> "FACTORY_NAME"
> ]
> }
> ],
> "measures": [
> {
> "name": "_COUNT_",
> "function": {
> "expression": "COUNT",
> "parameter": {
> "type": "constant",
> "value": "1",
> "next_parameter": null
> },
> "returntype": "bigint"
> },
> "dependent_measure_ref": null
> },
> {
> "name": "PV",
> "function": {
> "expression": "SUM",
> "parameter": {
> "type": "column",
> "value": "PV",
> "next_parameter": null
> },
> "returntype": "bigint"
> },
> "dependent_measure_ref": null
> },
> {
> "name": "USER_ID",
> "function": {
> "expression": "COUNT_DISTINCT",
> "parameter": {
> "type": "column",
> "value": "USER_ID",
> "next_parameter": null
> },
> "returntype": "bitmap"
> },
> "dependent_measure_ref": null
> }
> ],
> "rowkey": {
> "rowkey_columns": [
> {
> "column": "DT",
> "encoding": "dict"
> },
> {
> "column": "SPEC_ID",
> "encoding": "dict"
> },
> {
> "column": "SITE_ID",
> "encoding": "dict"
> },
> {
> "column": "DEALER_ID",
> "encoding": "dict"
> },
> {
> "column": "CITY_ID",
> "encoding": "dict"
> },
> {
> "column": "KEY_WORD",
> "encoding": "dict"
> },
> {
> "column": "REF_DOMAIN",
> "encoding": "dict"
> },
> {
> "column": "SITE_CLASS",
> "encoding": "dict"
> },
> {
> "column": "DEVICE_ID",
> "encoding": "dict"
> },
> {
> "column": "NEW_FLAG",
> "encoding": "dict"
> }
> ]
> },
> "signature": "8O48Zjj6YujqabUmbVcg+A==",
> "last_modified": 1459996931395,
> "model_name": "flw_test2",
> "null_string": null,
> "hbase_mapping": {
> "column_family": [
> {
> "name": "F1",
> "columns": [
> {
> "qualifier": "M",
> "measure_refs": [
> "_COUNT_",
> "PV"
> ]
> }
> ]
> },
> {
> "name": "F2",
> "columns": [
> {
> "qualifier": "M",
> "measure_refs": [
> "USER_ID"
> ]
> }
> ]
> }
> ]
> },
> "aggregation_groups": [
> {
> "includes": [
> "SPEC_ID",
> "CITY_ID",
> "DT",
> "SITE_CLASS",
> "SITE_ID"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": [
> [
> "SITE_CLASS",
> "CITY_ID",
> "SPEC_ID"
> ]
> ]
> }
> },
> {
> "includes": [
> "DT",
> "SITE_CLASS",
> "DEVICE_ID",
> "REF_DOMAIN",
> "SITE_ID"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": [
> [
> "DEVICE_ID",
> "REF_DOMAIN",
> "SITE_ID"
> ]
> ]
> }
> },
> {
> "includes": [
> "DT",
> "NEW_FLAG",
> "SPEC_ID",
> "REF_DOMAIN",
> "DEALER_ID"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": []
> }
> },
> {
> "includes": [
> "DT",
> "KEY_WORD",
> "SITE_ID",
> "DEVICE_ID",
> "REF_DOMAIN"
> ],
> "select_rule": {
> "hierarchy_dims": [],
> "mandatory_dims": [
> "DT"
> ],
> "joint_dims": []
> }
> }
> ],
> "notify_list": [],
> "status_need_notify": [],
> "partition_date_start": 1459814400000,
> "partition_date_end": 3153600000000,
> "auto_merge_time_ranges": [
> 604800000,
> 2419200000
> ],
> "retention_range": 0,
> "engine_type": 2,
> "storage_type": 2,
> "override_kylin_properties": {}
> }
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)