You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@kylin.apache.org by xl l <yo...@gmail.com> on 2017/02/16 08:10:08 UTC
kylin sql query with Weird error
see :
http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f21&sub=7BB3043221BA44E4BAF5760339280480
kylin 1.6 问题记录：
版本：apache-kylin-1.6.0-hbase1.x-bin
select
cast(SUM(pv) as double) as pv,
cast( count(distinct user_id) as double) as user_id
from olap.olap_log_accs_page_di
inner join DIM.DIM_LOG_USER_LOCATION on
DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_accs_page_di.location
inner join DIM.DIM_PUBLIC_DATE_INFO on
DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_di."DATE"
where
DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
group by
DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_LOCATION.user_region_name
order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
上面这个ｓｑｌ执行ＯＫ，符合预期。
但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
错误信息如下所示：
查看 kylin.log日志， 详细的异常信息 ：
http://note.youdao.com/noteshare?id=a1c257599774c4bccb0c6763923359d5&sub=11C6AA36AC894EDD9006DDAE17B16747
2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
ipc.CoprocessorRpcChannel:58 : Call failed on IOException
java.net.SocketTimeoutException: callTimeout=60000, callDuration=114625:
row ' ' on table 'KYLIN_F2H6NPOLR7' at
region=KYLIN_F2H6NPOLR7,,1487168742102.433e266be82448c5380610e9e7704658.,
hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
at
org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159)
at
org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.callExecService(RegionCoprocessorRpcChannel.java:95)
at
org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(CoprocessorRpcChannel.java:56)
at
org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos$CubeVisitService$Stub.visitCube(CubeVisitProtos.java:4178)
Caused by: java.lang.NullPointerException
at com.google.common.base.Preconditions.checkNotNull(Preconditions.java:191)
at
org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(HBaseReadonlyStore.java:131)
根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
附上该sql其他情况下：
如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select sum，count（distinct） 两个指标
只出现一个 指标， 也不会报错。
补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
附上 cube_desc 详细信息：
{ "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e", "last_modified":
1487147245924, "version": "1.6.0", "name":
"olap_log_accs_page_di_cube_0215", "model_name":
"olap_log_accs_page_di_cube_0215", "description":
"olap_log_accs_page_di_cube_0215", "null_string": null, "dimensions": [ {
"name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column": "YEAR",
"derived": null }, { "name": "QUARTER", "table":
"DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived": null }, {
"name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column": "MONTH_CN",
"derived": null }, { "name": "DATE", "table": "DIM.DIM_PUBLIC_DATE_INFO",
"column": "DATE", "derived": null }, { "name": "PROVINCE", "table":
"DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME", "derived": null }, {
"name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column": "CITY_NAME",
"derived": null }, { "name": "USER_COUNTRY", "table":
"DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME", "derived": null
}, { "name": "USER_REGION", "table": "DIM.DIM_LOG_USER_LOCATION", "column":
"USER_REGION_NAME", "derived": null }, { "name": "USER_PROVINCE", "table":
"DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME", "derived":
null }, { "name": "USER_CITY", "table": "DIM.DIM_LOG_USER_LOCATION",
"column": "USER_CITY_NAME", "derived": null }, { "name": "USER_TYPE",
"table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME", "derived": null },
{ "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN", "column":
"LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER", "table":
"DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME", "derived": null }, {
"name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
"BROWSER_NAME", "derived": null }, { "name": "APP_VERSION", "table":
"DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived": null } ],
"measures": [ { "name": "_COUNT_", "function": { "expression": "COUNT",
"parameter": { "type": "constant", "value": "1", "next_parameter": null },
"returntype": "bigint" }, "dependent_measure_ref": null }, { "name": "PV",
"function": { "expression": "SUM", "parameter": { "type": "column",
"value": "PV", "next_parameter": null }, "returntype": "bigint" },
"dependent_measure_ref": null }, { "name": "UV", "function": {
"expression": "COUNT_DISTINCT", "parameter": { "type": "column", "value":
"USER_ID", "next_parameter": null }, "returntype": "bitmap" },
"dependent_measure_ref": null }, { "name": "OUT_SESSION", "function": {
"expression": "SUM", "parameter": { "type": "column", "value":
"IS_OUT_SESSION", "next_parameter": null }, "returntype": "bigint" },
"dependent_measure_ref": null } ], "dictionaries": [], "rowkey": {
"rowkey_columns": [ { "column": "YEAR", "encoding": "dict", "isShardBy":
false }, { "column": "QUARTER_CN", "encoding": "dict", "isShardBy": false
}, { "column": "MONTH_CN", "encoding": "dict", "isShardBy": false }, {
"column": "DATE", "encoding": "dict", "isShardBy": false }, { "column":
"PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"TYPE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"LOGIN_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"REGISTER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"BROWSER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
"VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
"hbase_mapping": { "column_family": [ { "name": "F1", "columns": [ {
"qualifier": "M", "measure_refs": [ "_COUNT_", "PV", "OUT_SESSION" ] } ] },
{ "name": "F2", "columns": [ { "qualifier": "M", "measure_refs": [ "UV" ] }
] } ] }, "aggregation_groups": [ { "includes": [ "YEAR", "QUARTER_CN",
"MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME", "USER_COUNTRY_NAME",
"USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME", "TYPE_NAME",
"LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME" ],
"select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN", "MONTH_CN",
"DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [ "USER_COUNTRY_NAME",
"USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
"mandatory_dims": [], "joint_dims": [] } } ], "signature":
"odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [], "status_need_notify": [
"ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start": 1485907200000,
"partition_date_end": 3153600000000, "auto_merge_time_ranges": [ 604800000,
2419200000 ], "retention_range": 0, "engine_type": 2, "storage_type": 2,
"override_kylin_properties": { "kylin.cube.aggrgroup.max.combination":
"16384" } }
--
祝身体健康，万事如意 ^_^
Re: kylin sql query with Weird error
Posted by Billy Liu <bi...@apache.org>.
Glad to hear that.
2017-02-17 18:30 GMT+08:00 xl l <yo...@gmail.com>:
> hi，all：
> thanks for kyligence's help, I have finded the reason of this problem：
> hbase 1.1.2 bug: https://issues.apache.org/jira/browse/HBASE-14269
> After we upgrade the cluster to hbase 1.1.8, the sql query is ok.
> So the solution is to upgrade hbase version, thanks billyliu 。
>
> 2017-02-17 17:49 GMT+08:00 Billy Liu <bi...@apache.org>:
>
> > The Query works fine on the latest master, could not be reproduced. It
> > returns 255 records:
> >
> > Toys & Hobbies
> > 59.5137
> > 1
> > Toys & Hobbies
> > 87.3074
> > 1
> > Toys & Hobbies
> > 177.3848
> > 3
> > Toys & Hobbies
> > 59.9071
> > 2
> > Toys & Hobbies
> > 128.2840
> > 3
> >
> > 2017-02-17 12:40 GMT+08:00 xl l <yo...@gmail.com>:
> >
> > > hi,Billy Liu
> > > thanks. I did the same sql with sample cube, and the query result ｉｓ
> > > ａｌｓｏ ｅｒｒｏｒ .
> > >
> > > sql :
> > > select
> > > META_CATEG_NAME,
> > > sum(price) as total_selled,
> > > count(distinct seller_id) as sellers
> > > from kylin_sales
> > > inner join KYLIN_CATEGORY_GROUPINGS
> > > on KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID=KYLIN_SALES.LEAF_CATEG_ID
> > > and KYLIN_SALES.LSTG_SITE_ID = KYLIN_CATEGORY_GROUPINGS.SITE_ID
> > > where part_dt>='2012-01-01'
> > > and part_dt<='2013-01-01'
> > > and KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME in ('Toys &
> > Hobbies','Cameras
> > > & Photo')
> > > group by part_dt,KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME
> > >
> > >
> > > ｄｅｔａｉｌ ｄｅｓｃｒｉｐｔｉｏｎ ｓｅｅ ：
> > > http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f
> > 21&sub=
> > > 7BB3043221BA44E4BAF5760339280480
> > >
> > >
> > >
> > >
> > > 2017-02-16 23:27 GMT+08:00 Billy Liu <bi...@apache.org>:
> > >
> > > > Could you reproduce this issue on the sample cube? That would help
> the
> > > dev
> > > > team to identify the root cause quickly.
> > > >
> > > > 2017-02-16 20:30 GMT+08:00 xl l <yo...@gmail.com>:
> > > >
> > > > > HI, I am sure hbase is ok.
> > > > > 而且只有这个sql抛异常，且能稳定复现。 sql稍微改一下，就正常。
> > > > >
> > > > > 从异常日志看， 首先抛出异常的是
> > > > > Caused by: java.lang.NullPointerException
> > > > > at com.google.common.base.Preconditions.checkNotNull(
> > > > > Preconditions.java:191)
> > > > > at
> > > > > org.apache.kylin.storage.hbase.cube.v2.
> HBaseReadonlyStore$1$1.next(
> > > > > HBaseReadonlyStore.java:131)
> > > > >
> > > > > 对应于 代码：
> > > > >
> > > > > Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(i);
> > > > > Cell cell = findCell(oneRow, hbaseColumn.getFirst(),
> > > > > hbaseColumn.getSecond());
> > > > > Preconditions.checkNotNull(cell);
> > > > >
> > > > > cell 啥时候 会为空？
> > > > >
> > > > >
> > > > >
> > > > >
> > > > >
> > > > > 2017-02-16 17:47 GMT+08:00 ShaoFeng Shi <sh...@apache.org>:
> > > > >
> > > > > > "java.net.SocketTimeoutException", did you check HBase's healthy
> > > > status?
> > > > > > Are all regions of the table "KYLIN_F2H6NPOLR7" online?
> > > > > >
> > > > > > 2017-02-16 16:10 GMT+08:00 xl l <yo...@gmail.com>:
> > > > > >
> > > > > > > see :
> > > > > > >
> > > > > > > http://note.youdao.com/noteshare?id=
> > df34c64dcf3cf801a9c085be0c3f5f
> > > > > > 21&sub=
> > > > > > > 7BB3043221BA44E4BAF5760339280480
> > > > > > >
> > > > > > >
> > > > > > > kylin 1.6 问题记录：
> > > > > > > 版本：apache-kylin-1.6.0-hbase1.x-bin
> > > > > > >
> > > > > > >
> > > > > > > select
> > > > > > > cast(SUM(pv) as double) as pv,
> > > > > > > cast( count(distinct user_id) as double) as user_id
> > > > > > > from olap.olap_log_accs_page_di
> > > > > > > inner join DIM.DIM_LOG_USER_LOCATION on
> > > > > > > DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_
> > > > > > > accs_page_di.location
> > > > > > >
> > > > > > > inner join DIM.DIM_PUBLIC_DATE_INFO on
> > > > > > > DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_
> > di."DATE"
> > > > > > > where
> > > > > > > DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
> > > > > > > and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
> > > > > > > and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
> > > > > > > group by
> > > > > > > DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_
> > > > > > > USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_
> > > > > > > LOCATION.user_region_name
> > > > > > >
> > > > > > > order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
> > > > > > >
> > > > > > > 上面这个ｓｑｌ执行ＯＫ，符合预期。
> > > > > > > 但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
> > > > > > >
> > > > > > >
> > > > > > > 错误信息如下所示：
> > > > > > >
> > > > > > >
> > > > > > > 查看 kylin.log日志， 详细的异常信息 ：
> > > > > > > http://note.youdao.com/noteshare?id=
> > a1c257599774c4bccb0c6763923359
> > > > > > d5&sub=
> > > > > > > 11C6AA36AC894EDD9006DDAE17B16747
> > > > > > > 2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
> > > > > > > ipc.CoprocessorRpcChannel:58 : Call failed on IOException
> > > > > > > java.net.SocketTimeoutException: callTimeout=60000,
> > > > > callDuration=114625:
> > > > > > > row ' ' on table 'KYLIN_F2H6NPOLR7' at
> > > > > > > region=KYLIN_F2H6NPOLR7,,1487168742102.
> > > > 433e266be82448c5380610e9e77046
> > > > > > 58.,
> > > > > > > hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
> > > > > > > at
> > > > > > > org.apache.hadoop.hbase.client.RpcRetryingCaller.
> > callWithRetries(
> > > > > > > RpcRetryingCaller.java:159)
> > > > > > > at
> > > > > > > org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.
> > > > > callExecService(
> > > > > > > RegionCoprocessorRpcChannel.java:95)
> > > > > > > at
> > > > > > > org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(
> > > > > > > CoprocessorRpcChannel.java:56)
> > > > > > > at
> > > > > > > org.apache.kylin.storage.hbase.cube.v2.coprocessor.
> > > > endpoint.generated.
> > > > > > > CubeVisitProtos$CubeVisitService$Stub.
> visitCube(CubeVisitProtos.
> > > > > > java:4178)
> > > > > > >
> > > > > > > Caused by: java.lang.NullPointerException
> > > > > > > at com.google.common.base.Preconditions.checkNotNull(
> > > > > > > Preconditions.java:191)
> > > > > > > at
> > > > > > > org.apache.kylin.storage.hbase.cube.v2.
> > > HBaseReadonlyStore$1$1.next(
> > > > > > > HBaseReadonlyStore.java:131)
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > > 根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
> > > > > > >
> > > > > > > 麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > > 附上该sql其他情况下：
> > > > > > > 如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select
> > > > sum，count（distinct）
> > > > > > 两个指标
> > > > > > > 只出现一个 指标， 也不会报错。
> > > > > > >
> > > > > > > 补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
> > > > > > >
> > > > > > >
> > > > > > > 附上 cube_desc 详细信息：
> > > > > > >
> > > > > > > { "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e",
> > "last_modified":
> > > > > > > 1487147245924, "version": "1.6.0", "name":
> > > > > > > "olap_log_accs_page_di_cube_0215", "model_name":
> > > > > > > "olap_log_accs_page_di_cube_0215", "description":
> > > > > > > "olap_log_accs_page_di_cube_0215", "null_string": null,
> > > > "dimensions":
> > > > > [
> > > > > > {
> > > > > > > "name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > > > "YEAR",
> > > > > > > "derived": null }, { "name": "QUARTER", "table":
> > > > > > > "DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived":
> > null
> > > > },
> > > > > {
> > > > > > > "name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > > > > > "MONTH_CN",
> > > > > > > "derived": null }, { "name": "DATE", "table":
> > > > > "DIM.DIM_PUBLIC_DATE_INFO",
> > > > > > > "column": "DATE", "derived": null }, { "name": "PROVINCE",
> > "table":
> > > > > > > "DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME",
> "derived":
> > > > null
> > > > > > }, {
> > > > > > > "name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column":
> > > > > > "CITY_NAME",
> > > > > > > "derived": null }, { "name": "USER_COUNTRY", "table":
> > > > > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME",
> > > > "derived":
> > > > > > null
> > > > > > > }, { "name": "USER_REGION", "table":
> "DIM.DIM_LOG_USER_LOCATION",
> > > > > > "column":
> > > > > > > "USER_REGION_NAME", "derived": null }, { "name":
> "USER_PROVINCE",
> > > > > > "table":
> > > > > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME",
> > > > "derived":
> > > > > > > null }, { "name": "USER_CITY", "table":
> > > "DIM.DIM_LOG_USER_LOCATION",
> > > > > > > "column": "USER_CITY_NAME", "derived": null }, { "name":
> > > "USER_TYPE",
> > > > > > > "table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME",
> > "derived":
> > > > > null
> > > > > > },
> > > > > > > { "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN",
> "column":
> > > > > > > "LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER",
> > "table":
> > > > > > > "DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME",
> "derived":
> > > null
> > > > > },
> > > > > > {
> > > > > > > "name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
> > > > > > > "BROWSER_NAME", "derived": null }, { "name": "APP_VERSION",
> > > "table":
> > > > > > > "DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived":
> > > null
> > > > }
> > > > > ],
> > > > > > > "measures": [ { "name": "_COUNT_", "function": { "expression":
> > > > "COUNT",
> > > > > > > "parameter": { "type": "constant", "value": "1",
> > "next_parameter":
> > > > null
> > > > > > },
> > > > > > > "returntype": "bigint" }, "dependent_measure_ref": null }, {
> > > "name":
> > > > > > "PV",
> > > > > > > "function": { "expression": "SUM", "parameter": { "type":
> > "column",
> > > > > > > "value": "PV", "next_parameter": null }, "returntype": "bigint"
> > },
> > > > > > > "dependent_measure_ref": null }, { "name": "UV", "function": {
> > > > > > > "expression": "COUNT_DISTINCT", "parameter": { "type":
> "column",
> > > > > "value":
> > > > > > > "USER_ID", "next_parameter": null }, "returntype": "bitmap" },
> > > > > > > "dependent_measure_ref": null }, { "name": "OUT_SESSION",
> > > > "function": {
> > > > > > > "expression": "SUM", "parameter": { "type": "column", "value":
> > > > > > > "IS_OUT_SESSION", "next_parameter": null }, "returntype":
> > "bigint"
> > > },
> > > > > > > "dependent_measure_ref": null } ], "dictionaries": [],
> "rowkey":
> > {
> > > > > > > "rowkey_columns": [ { "column": "YEAR", "encoding": "dict",
> > > > > "isShardBy":
> > > > > > > false }, { "column": "QUARTER_CN", "encoding": "dict",
> > "isShardBy":
> > > > > false
> > > > > > > }, { "column": "MONTH_CN", "encoding": "dict", "isShardBy":
> false
> > > },
> > > > {
> > > > > > > "column": "DATE", "encoding": "dict", "isShardBy": false }, {
> > > > "column":
> > > > > > > "PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > "column":
> > > > > > > "CITY_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > > > "USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false },
> {
> > > > > > "column":
> > > > > > > "USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > > "column":
> > > > > > > "USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false
> }, {
> > > > > > "column":
> > > > > > > "USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > "column":
> > > > > > > "TYPE_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > > > "LOGIN_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > > > "REGISTER_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > "column":
> > > > > > > "BROWSER_NAME", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > > > > "VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
> > > > > > > "hbase_mapping": { "column_family": [ { "name": "F1",
> "columns":
> > [
> > > {
> > > > > > > "qualifier": "M", "measure_refs": [ "_COUNT_", "PV",
> > "OUT_SESSION"
> > > ]
> > > > }
> > > > > ]
> > > > > > },
> > > > > > > { "name": "F2", "columns": [ { "qualifier": "M",
> "measure_refs":
> > [
> > > > "UV"
> > > > > > ] }
> > > > > > > ] } ] }, "aggregation_groups": [ { "includes": [ "YEAR",
> > > > "QUARTER_CN",
> > > > > > > "MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME",
> > > > "USER_COUNTRY_NAME",
> > > > > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME",
> > > > > "TYPE_NAME",
> > > > > > > "LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME"
> ],
> > > > > > > "select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN",
> > > > > "MONTH_CN",
> > > > > > > "DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [
> > "USER_COUNTRY_NAME",
> > > > > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
> > > > > > > "mandatory_dims": [], "joint_dims": [] } } ], "signature":
> > > > > > > "odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [],
> > > "status_need_notify":
> > > > [
> > > > > > > "ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start":
> > > > > 1485907200000,
> > > > > > > "partition_date_end": 3153600000000, "auto_merge_time_ranges":
> [
> > > > > > 604800000,
> > > > > > > 2419200000 ], "retention_range": 0, "engine_type": 2,
> > > "storage_type":
> > > > > 2,
> > > > > > > "override_kylin_properties": { "kylin.cube.aggrgroup.max.
> > > > combination":
> > > > > > > "16384" } }
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > > --
> > > > > > > 祝身体健康，万事如意 ^_^
> > > > > > >
> > > > > >
> > > > > >
> > > > > >
> > > > > > --
> > > > > > Best regards,
> > > > > >
> > > > > > Shaofeng Shi 史少锋
> > > > > >
> > > > >
> > > > >
> > > > >
> > > > > --
> > > > > 祝身体健康，万事如意 ^_^
> > > > >
> > > >
> > >
> > >
> > >
> > > --
> > > 祝身体健康，万事如意 ^_^
> > >
> >
>
>
>
> --
> 祝身体健康，万事如意 ^_^
>
Re: kylin sql query with Weird error
Posted by xl l <yo...@gmail.com>.
hi，all：
thanks for kyligence's help, I have finded the reason of this problem：
hbase 1.1.2 bug: https://issues.apache.org/jira/browse/HBASE-14269
After we upgrade the cluster to hbase 1.1.8, the sql query is ok.
So the solution is to upgrade hbase version, thanks billyliu 。
2017-02-17 17:49 GMT+08:00 Billy Liu <bi...@apache.org>:
> The Query works fine on the latest master, could not be reproduced. It
> returns 255 records:
>
> Toys & Hobbies
> 59.5137
> 1
> Toys & Hobbies
> 87.3074
> 1
> Toys & Hobbies
> 177.3848
> 3
> Toys & Hobbies
> 59.9071
> 2
> Toys & Hobbies
> 128.2840
> 3
>
> 2017-02-17 12:40 GMT+08:00 xl l <yo...@gmail.com>:
>
> > hi,Billy Liu
> > thanks. I did the same sql with sample cube, and the query result ｉｓ
> > ａｌｓｏ ｅｒｒｏｒ .
> >
> > sql :
> > select
> > META_CATEG_NAME,
> > sum(price) as total_selled,
> > count(distinct seller_id) as sellers
> > from kylin_sales
> > inner join KYLIN_CATEGORY_GROUPINGS
> > on KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID=KYLIN_SALES.LEAF_CATEG_ID
> > and KYLIN_SALES.LSTG_SITE_ID = KYLIN_CATEGORY_GROUPINGS.SITE_ID
> > where part_dt>='2012-01-01'
> > and part_dt<='2013-01-01'
> > and KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME in ('Toys &
> Hobbies','Cameras
> > & Photo')
> > group by part_dt,KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME
> >
> >
> > ｄｅｔａｉｌ ｄｅｓｃｒｉｐｔｉｏｎ ｓｅｅ ：
> > http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f
> 21&sub=
> > 7BB3043221BA44E4BAF5760339280480
> >
> >
> >
> >
> > 2017-02-16 23:27 GMT+08:00 Billy Liu <bi...@apache.org>:
> >
> > > Could you reproduce this issue on the sample cube? That would help the
> > dev
> > > team to identify the root cause quickly.
> > >
> > > 2017-02-16 20:30 GMT+08:00 xl l <yo...@gmail.com>:
> > >
> > > > HI, I am sure hbase is ok.
> > > > 而且只有这个sql抛异常，且能稳定复现。 sql稍微改一下，就正常。
> > > >
> > > > 从异常日志看， 首先抛出异常的是
> > > > Caused by: java.lang.NullPointerException
> > > > at com.google.common.base.Preconditions.checkNotNull(
> > > > Preconditions.java:191)
> > > > at
> > > > org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> > > > HBaseReadonlyStore.java:131)
> > > >
> > > > 对应于 代码：
> > > >
> > > > Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(i);
> > > > Cell cell = findCell(oneRow, hbaseColumn.getFirst(),
> > > > hbaseColumn.getSecond());
> > > > Preconditions.checkNotNull(cell);
> > > >
> > > > cell 啥时候 会为空？
> > > >
> > > >
> > > >
> > > >
> > > >
> > > > 2017-02-16 17:47 GMT+08:00 ShaoFeng Shi <sh...@apache.org>:
> > > >
> > > > > "java.net.SocketTimeoutException", did you check HBase's healthy
> > > status?
> > > > > Are all regions of the table "KYLIN_F2H6NPOLR7" online?
> > > > >
> > > > > 2017-02-16 16:10 GMT+08:00 xl l <yo...@gmail.com>:
> > > > >
> > > > > > see :
> > > > > >
> > > > > > http://note.youdao.com/noteshare?id=
> df34c64dcf3cf801a9c085be0c3f5f
> > > > > 21&sub=
> > > > > > 7BB3043221BA44E4BAF5760339280480
> > > > > >
> > > > > >
> > > > > > kylin 1.6 问题记录：
> > > > > > 版本：apache-kylin-1.6.0-hbase1.x-bin
> > > > > >
> > > > > >
> > > > > > select
> > > > > > cast(SUM(pv) as double) as pv,
> > > > > > cast( count(distinct user_id) as double) as user_id
> > > > > > from olap.olap_log_accs_page_di
> > > > > > inner join DIM.DIM_LOG_USER_LOCATION on
> > > > > > DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_
> > > > > > accs_page_di.location
> > > > > >
> > > > > > inner join DIM.DIM_PUBLIC_DATE_INFO on
> > > > > > DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_
> di."DATE"
> > > > > > where
> > > > > > DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
> > > > > > and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
> > > > > > and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
> > > > > > group by
> > > > > > DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_
> > > > > > USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_
> > > > > > LOCATION.user_region_name
> > > > > >
> > > > > > order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
> > > > > >
> > > > > > 上面这个ｓｑｌ执行ＯＫ，符合预期。
> > > > > > 但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
> > > > > >
> > > > > >
> > > > > > 错误信息如下所示：
> > > > > >
> > > > > >
> > > > > > 查看 kylin.log日志， 详细的异常信息 ：
> > > > > > http://note.youdao.com/noteshare?id=
> a1c257599774c4bccb0c6763923359
> > > > > d5&sub=
> > > > > > 11C6AA36AC894EDD9006DDAE17B16747
> > > > > > 2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
> > > > > > ipc.CoprocessorRpcChannel:58 : Call failed on IOException
> > > > > > java.net.SocketTimeoutException: callTimeout=60000,
> > > > callDuration=114625:
> > > > > > row ' ' on table 'KYLIN_F2H6NPOLR7' at
> > > > > > region=KYLIN_F2H6NPOLR7,,1487168742102.
> > > 433e266be82448c5380610e9e77046
> > > > > 58.,
> > > > > > hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
> > > > > > at
> > > > > > org.apache.hadoop.hbase.client.RpcRetryingCaller.
> callWithRetries(
> > > > > > RpcRetryingCaller.java:159)
> > > > > > at
> > > > > > org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.
> > > > callExecService(
> > > > > > RegionCoprocessorRpcChannel.java:95)
> > > > > > at
> > > > > > org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(
> > > > > > CoprocessorRpcChannel.java:56)
> > > > > > at
> > > > > > org.apache.kylin.storage.hbase.cube.v2.coprocessor.
> > > endpoint.generated.
> > > > > > CubeVisitProtos$CubeVisitService$Stub.visitCube(CubeVisitProtos.
> > > > > java:4178)
> > > > > >
> > > > > > Caused by: java.lang.NullPointerException
> > > > > > at com.google.common.base.Preconditions.checkNotNull(
> > > > > > Preconditions.java:191)
> > > > > > at
> > > > > > org.apache.kylin.storage.hbase.cube.v2.
> > HBaseReadonlyStore$1$1.next(
> > > > > > HBaseReadonlyStore.java:131)
> > > > > >
> > > > > >
> > > > > >
> > > > > > 根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
> > > > > >
> > > > > > 麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
> > > > > >
> > > > > >
> > > > > >
> > > > > >
> > > > > > 附上该sql其他情况下：
> > > > > > 如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select
> > > sum，count（distinct）
> > > > > 两个指标
> > > > > > 只出现一个 指标， 也不会报错。
> > > > > >
> > > > > > 补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
> > > > > >
> > > > > >
> > > > > > 附上 cube_desc 详细信息：
> > > > > >
> > > > > > { "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e",
> "last_modified":
> > > > > > 1487147245924, "version": "1.6.0", "name":
> > > > > > "olap_log_accs_page_di_cube_0215", "model_name":
> > > > > > "olap_log_accs_page_di_cube_0215", "description":
> > > > > > "olap_log_accs_page_di_cube_0215", "null_string": null,
> > > "dimensions":
> > > > [
> > > > > {
> > > > > > "name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > > "YEAR",
> > > > > > "derived": null }, { "name": "QUARTER", "table":
> > > > > > "DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived":
> null
> > > },
> > > > {
> > > > > > "name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > > > > "MONTH_CN",
> > > > > > "derived": null }, { "name": "DATE", "table":
> > > > "DIM.DIM_PUBLIC_DATE_INFO",
> > > > > > "column": "DATE", "derived": null }, { "name": "PROVINCE",
> "table":
> > > > > > "DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME", "derived":
> > > null
> > > > > }, {
> > > > > > "name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column":
> > > > > "CITY_NAME",
> > > > > > "derived": null }, { "name": "USER_COUNTRY", "table":
> > > > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME",
> > > "derived":
> > > > > null
> > > > > > }, { "name": "USER_REGION", "table": "DIM.DIM_LOG_USER_LOCATION",
> > > > > "column":
> > > > > > "USER_REGION_NAME", "derived": null }, { "name": "USER_PROVINCE",
> > > > > "table":
> > > > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME",
> > > "derived":
> > > > > > null }, { "name": "USER_CITY", "table":
> > "DIM.DIM_LOG_USER_LOCATION",
> > > > > > "column": "USER_CITY_NAME", "derived": null }, { "name":
> > "USER_TYPE",
> > > > > > "table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME",
> "derived":
> > > > null
> > > > > },
> > > > > > { "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN", "column":
> > > > > > "LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER",
> "table":
> > > > > > "DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME", "derived":
> > null
> > > > },
> > > > > {
> > > > > > "name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
> > > > > > "BROWSER_NAME", "derived": null }, { "name": "APP_VERSION",
> > "table":
> > > > > > "DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived":
> > null
> > > }
> > > > ],
> > > > > > "measures": [ { "name": "_COUNT_", "function": { "expression":
> > > "COUNT",
> > > > > > "parameter": { "type": "constant", "value": "1",
> "next_parameter":
> > > null
> > > > > },
> > > > > > "returntype": "bigint" }, "dependent_measure_ref": null }, {
> > "name":
> > > > > "PV",
> > > > > > "function": { "expression": "SUM", "parameter": { "type":
> "column",
> > > > > > "value": "PV", "next_parameter": null }, "returntype": "bigint"
> },
> > > > > > "dependent_measure_ref": null }, { "name": "UV", "function": {
> > > > > > "expression": "COUNT_DISTINCT", "parameter": { "type": "column",
> > > > "value":
> > > > > > "USER_ID", "next_parameter": null }, "returntype": "bitmap" },
> > > > > > "dependent_measure_ref": null }, { "name": "OUT_SESSION",
> > > "function": {
> > > > > > "expression": "SUM", "parameter": { "type": "column", "value":
> > > > > > "IS_OUT_SESSION", "next_parameter": null }, "returntype":
> "bigint"
> > },
> > > > > > "dependent_measure_ref": null } ], "dictionaries": [], "rowkey":
> {
> > > > > > "rowkey_columns": [ { "column": "YEAR", "encoding": "dict",
> > > > "isShardBy":
> > > > > > false }, { "column": "QUARTER_CN", "encoding": "dict",
> "isShardBy":
> > > > false
> > > > > > }, { "column": "MONTH_CN", "encoding": "dict", "isShardBy": false
> > },
> > > {
> > > > > > "column": "DATE", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > > > "PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > > > "CITY_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > > > "USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > > "column":
> > > > > > "USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > "column":
> > > > > > "USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > > "column":
> > > > > > "USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > > > "TYPE_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > > > "LOGIN_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > > > "REGISTER_NAME", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > > > "BROWSER_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > > "VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
> > > > > > "hbase_mapping": { "column_family": [ { "name": "F1", "columns":
> [
> > {
> > > > > > "qualifier": "M", "measure_refs": [ "_COUNT_", "PV",
> "OUT_SESSION"
> > ]
> > > }
> > > > ]
> > > > > },
> > > > > > { "name": "F2", "columns": [ { "qualifier": "M", "measure_refs":
> [
> > > "UV"
> > > > > ] }
> > > > > > ] } ] }, "aggregation_groups": [ { "includes": [ "YEAR",
> > > "QUARTER_CN",
> > > > > > "MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME",
> > > "USER_COUNTRY_NAME",
> > > > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME",
> > > > "TYPE_NAME",
> > > > > > "LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME" ],
> > > > > > "select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN",
> > > > "MONTH_CN",
> > > > > > "DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [
> "USER_COUNTRY_NAME",
> > > > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
> > > > > > "mandatory_dims": [], "joint_dims": [] } } ], "signature":
> > > > > > "odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [],
> > "status_need_notify":
> > > [
> > > > > > "ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start":
> > > > 1485907200000,
> > > > > > "partition_date_end": 3153600000000, "auto_merge_time_ranges": [
> > > > > 604800000,
> > > > > > 2419200000 ], "retention_range": 0, "engine_type": 2,
> > "storage_type":
> > > > 2,
> > > > > > "override_kylin_properties": { "kylin.cube.aggrgroup.max.
> > > combination":
> > > > > > "16384" } }
> > > > > >
> > > > > >
> > > > > >
> > > > > >
> > > > > >
> > > > > >
> > > > > > --
> > > > > > 祝身体健康，万事如意 ^_^
> > > > > >
> > > > >
> > > > >
> > > > >
> > > > > --
> > > > > Best regards,
> > > > >
> > > > > Shaofeng Shi 史少锋
> > > > >
> > > >
> > > >
> > > >
> > > > --
> > > > 祝身体健康，万事如意 ^_^
> > > >
> > >
> >
> >
> >
> > --
> > 祝身体健康，万事如意 ^_^
> >
>
--
祝身体健康，万事如意 ^_^
Re: kylin sql query with Weird error
Posted by Billy Liu <bi...@apache.org>.
The Query works fine on the latest master, could not be reproduced. It
returns 255 records:
Toys & Hobbies
59.5137
1
Toys & Hobbies
87.3074
1
Toys & Hobbies
177.3848
3
Toys & Hobbies
59.9071
2
Toys & Hobbies
128.2840
3
2017-02-17 12:40 GMT+08:00 xl l <yo...@gmail.com>:
> hi,Billy Liu
> thanks. I did the same sql with sample cube, and the query result ｉｓ
> ａｌｓｏ ｅｒｒｏｒ .
>
> sql :
> select
> META_CATEG_NAME,
> sum(price) as total_selled,
> count(distinct seller_id) as sellers
> from kylin_sales
> inner join KYLIN_CATEGORY_GROUPINGS
> on KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID=KYLIN_SALES.LEAF_CATEG_ID
> and KYLIN_SALES.LSTG_SITE_ID = KYLIN_CATEGORY_GROUPINGS.SITE_ID
> where part_dt>='2012-01-01'
> and part_dt<='2013-01-01'
> and KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME in ('Toys & Hobbies','Cameras
> & Photo')
> group by part_dt,KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME
>
>
> ｄｅｔａｉｌ ｄｅｓｃｒｉｐｔｉｏｎ ｓｅｅ ：
> http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f21&sub=
> 7BB3043221BA44E4BAF5760339280480
>
>
>
>
> 2017-02-16 23:27 GMT+08:00 Billy Liu <bi...@apache.org>:
>
> > Could you reproduce this issue on the sample cube? That would help the
> dev
> > team to identify the root cause quickly.
> >
> > 2017-02-16 20:30 GMT+08:00 xl l <yo...@gmail.com>:
> >
> > > HI, I am sure hbase is ok.
> > > 而且只有这个sql抛异常，且能稳定复现。 sql稍微改一下，就正常。
> > >
> > > 从异常日志看， 首先抛出异常的是
> > > Caused by: java.lang.NullPointerException
> > > at com.google.common.base.Preconditions.checkNotNull(
> > > Preconditions.java:191)
> > > at
> > > org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> > > HBaseReadonlyStore.java:131)
> > >
> > > 对应于 代码：
> > >
> > > Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(i);
> > > Cell cell = findCell(oneRow, hbaseColumn.getFirst(),
> > > hbaseColumn.getSecond());
> > > Preconditions.checkNotNull(cell);
> > >
> > > cell 啥时候 会为空？
> > >
> > >
> > >
> > >
> > >
> > > 2017-02-16 17:47 GMT+08:00 ShaoFeng Shi <sh...@apache.org>:
> > >
> > > > "java.net.SocketTimeoutException", did you check HBase's healthy
> > status?
> > > > Are all regions of the table "KYLIN_F2H6NPOLR7" online?
> > > >
> > > > 2017-02-16 16:10 GMT+08:00 xl l <yo...@gmail.com>:
> > > >
> > > > > see :
> > > > >
> > > > > http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f
> > > > 21&sub=
> > > > > 7BB3043221BA44E4BAF5760339280480
> > > > >
> > > > >
> > > > > kylin 1.6 问题记录：
> > > > > 版本：apache-kylin-1.6.0-hbase1.x-bin
> > > > >
> > > > >
> > > > > select
> > > > > cast(SUM(pv) as double) as pv,
> > > > > cast( count(distinct user_id) as double) as user_id
> > > > > from olap.olap_log_accs_page_di
> > > > > inner join DIM.DIM_LOG_USER_LOCATION on
> > > > > DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_
> > > > > accs_page_di.location
> > > > >
> > > > > inner join DIM.DIM_PUBLIC_DATE_INFO on
> > > > > DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_di."DATE"
> > > > > where
> > > > > DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
> > > > > and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
> > > > > and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
> > > > > group by
> > > > > DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_
> > > > > USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_
> > > > > LOCATION.user_region_name
> > > > >
> > > > > order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
> > > > >
> > > > > 上面这个ｓｑｌ执行ＯＫ，符合预期。
> > > > > 但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
> > > > >
> > > > >
> > > > > 错误信息如下所示：
> > > > >
> > > > >
> > > > > 查看 kylin.log日志， 详细的异常信息 ：
> > > > > http://note.youdao.com/noteshare?id=a1c257599774c4bccb0c6763923359
> > > > d5&sub=
> > > > > 11C6AA36AC894EDD9006DDAE17B16747
> > > > > 2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
> > > > > ipc.CoprocessorRpcChannel:58 : Call failed on IOException
> > > > > java.net.SocketTimeoutException: callTimeout=60000,
> > > callDuration=114625:
> > > > > row ' ' on table 'KYLIN_F2H6NPOLR7' at
> > > > > region=KYLIN_F2H6NPOLR7,,1487168742102.
> > 433e266be82448c5380610e9e77046
> > > > 58.,
> > > > > hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
> > > > > at
> > > > > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(
> > > > > RpcRetryingCaller.java:159)
> > > > > at
> > > > > org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.
> > > callExecService(
> > > > > RegionCoprocessorRpcChannel.java:95)
> > > > > at
> > > > > org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(
> > > > > CoprocessorRpcChannel.java:56)
> > > > > at
> > > > > org.apache.kylin.storage.hbase.cube.v2.coprocessor.
> > endpoint.generated.
> > > > > CubeVisitProtos$CubeVisitService$Stub.visitCube(CubeVisitProtos.
> > > > java:4178)
> > > > >
> > > > > Caused by: java.lang.NullPointerException
> > > > > at com.google.common.base.Preconditions.checkNotNull(
> > > > > Preconditions.java:191)
> > > > > at
> > > > > org.apache.kylin.storage.hbase.cube.v2.
> HBaseReadonlyStore$1$1.next(
> > > > > HBaseReadonlyStore.java:131)
> > > > >
> > > > >
> > > > >
> > > > > 根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
> > > > >
> > > > > 麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
> > > > >
> > > > >
> > > > >
> > > > >
> > > > > 附上该sql其他情况下：
> > > > > 如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select
> > sum，count（distinct）
> > > > 两个指标
> > > > > 只出现一个 指标， 也不会报错。
> > > > >
> > > > > 补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
> > > > >
> > > > >
> > > > > 附上 cube_desc 详细信息：
> > > > >
> > > > > { "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e", "last_modified":
> > > > > 1487147245924, "version": "1.6.0", "name":
> > > > > "olap_log_accs_page_di_cube_0215", "model_name":
> > > > > "olap_log_accs_page_di_cube_0215", "description":
> > > > > "olap_log_accs_page_di_cube_0215", "null_string": null,
> > "dimensions":
> > > [
> > > > {
> > > > > "name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > "YEAR",
> > > > > "derived": null }, { "name": "QUARTER", "table":
> > > > > "DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived": null
> > },
> > > {
> > > > > "name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > > > "MONTH_CN",
> > > > > "derived": null }, { "name": "DATE", "table":
> > > "DIM.DIM_PUBLIC_DATE_INFO",
> > > > > "column": "DATE", "derived": null }, { "name": "PROVINCE", "table":
> > > > > "DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME", "derived":
> > null
> > > > }, {
> > > > > "name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column":
> > > > "CITY_NAME",
> > > > > "derived": null }, { "name": "USER_COUNTRY", "table":
> > > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME",
> > "derived":
> > > > null
> > > > > }, { "name": "USER_REGION", "table": "DIM.DIM_LOG_USER_LOCATION",
> > > > "column":
> > > > > "USER_REGION_NAME", "derived": null }, { "name": "USER_PROVINCE",
> > > > "table":
> > > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME",
> > "derived":
> > > > > null }, { "name": "USER_CITY", "table":
> "DIM.DIM_LOG_USER_LOCATION",
> > > > > "column": "USER_CITY_NAME", "derived": null }, { "name":
> "USER_TYPE",
> > > > > "table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME", "derived":
> > > null
> > > > },
> > > > > { "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN", "column":
> > > > > "LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER", "table":
> > > > > "DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME", "derived":
> null
> > > },
> > > > {
> > > > > "name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
> > > > > "BROWSER_NAME", "derived": null }, { "name": "APP_VERSION",
> "table":
> > > > > "DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived":
> null
> > }
> > > ],
> > > > > "measures": [ { "name": "_COUNT_", "function": { "expression":
> > "COUNT",
> > > > > "parameter": { "type": "constant", "value": "1", "next_parameter":
> > null
> > > > },
> > > > > "returntype": "bigint" }, "dependent_measure_ref": null }, {
> "name":
> > > > "PV",
> > > > > "function": { "expression": "SUM", "parameter": { "type": "column",
> > > > > "value": "PV", "next_parameter": null }, "returntype": "bigint" },
> > > > > "dependent_measure_ref": null }, { "name": "UV", "function": {
> > > > > "expression": "COUNT_DISTINCT", "parameter": { "type": "column",
> > > "value":
> > > > > "USER_ID", "next_parameter": null }, "returntype": "bitmap" },
> > > > > "dependent_measure_ref": null }, { "name": "OUT_SESSION",
> > "function": {
> > > > > "expression": "SUM", "parameter": { "type": "column", "value":
> > > > > "IS_OUT_SESSION", "next_parameter": null }, "returntype": "bigint"
> },
> > > > > "dependent_measure_ref": null } ], "dictionaries": [], "rowkey": {
> > > > > "rowkey_columns": [ { "column": "YEAR", "encoding": "dict",
> > > "isShardBy":
> > > > > false }, { "column": "QUARTER_CN", "encoding": "dict", "isShardBy":
> > > false
> > > > > }, { "column": "MONTH_CN", "encoding": "dict", "isShardBy": false
> },
> > {
> > > > > "column": "DATE", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > "PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > "CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > > > "USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > "column":
> > > > > "USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > > "USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> > > > "column":
> > > > > "USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > "TYPE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > > > "LOGIN_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > > > "REGISTER_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > > "BROWSER_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > > "VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
> > > > > "hbase_mapping": { "column_family": [ { "name": "F1", "columns": [
> {
> > > > > "qualifier": "M", "measure_refs": [ "_COUNT_", "PV", "OUT_SESSION"
> ]
> > }
> > > ]
> > > > },
> > > > > { "name": "F2", "columns": [ { "qualifier": "M", "measure_refs": [
> > "UV"
> > > > ] }
> > > > > ] } ] }, "aggregation_groups": [ { "includes": [ "YEAR",
> > "QUARTER_CN",
> > > > > "MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME",
> > "USER_COUNTRY_NAME",
> > > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME",
> > > "TYPE_NAME",
> > > > > "LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME" ],
> > > > > "select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN",
> > > "MONTH_CN",
> > > > > "DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [ "USER_COUNTRY_NAME",
> > > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
> > > > > "mandatory_dims": [], "joint_dims": [] } } ], "signature":
> > > > > "odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [],
> "status_need_notify":
> > [
> > > > > "ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start":
> > > 1485907200000,
> > > > > "partition_date_end": 3153600000000, "auto_merge_time_ranges": [
> > > > 604800000,
> > > > > 2419200000 ], "retention_range": 0, "engine_type": 2,
> "storage_type":
> > > 2,
> > > > > "override_kylin_properties": { "kylin.cube.aggrgroup.max.
> > combination":
> > > > > "16384" } }
> > > > >
> > > > >
> > > > >
> > > > >
> > > > >
> > > > >
> > > > > --
> > > > > 祝身体健康，万事如意 ^_^
> > > > >
> > > >
> > > >
> > > >
> > > > --
> > > > Best regards,
> > > >
> > > > Shaofeng Shi 史少锋
> > > >
> > >
> > >
> > >
> > > --
> > > 祝身体健康，万事如意 ^_^
> > >
> >
>
>
>
> --
> 祝身体健康，万事如意 ^_^
>
Re: kylin sql query with Weird error
Posted by xl l <yo...@gmail.com>.
hi,Billy Liu
thanks. I did the same sql with sample cube, and the query result ｉｓ
ａｌｓｏ ｅｒｒｏｒ .
sql :
select
META_CATEG_NAME,
sum(price) as total_selled,
count(distinct seller_id) as sellers
from kylin_sales
inner join KYLIN_CATEGORY_GROUPINGS
on KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID=KYLIN_SALES.LEAF_CATEG_ID
and KYLIN_SALES.LSTG_SITE_ID = KYLIN_CATEGORY_GROUPINGS.SITE_ID
where part_dt>='2012-01-01'
and part_dt<='2013-01-01'
and KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME in ('Toys & Hobbies','Cameras
& Photo')
group by part_dt,KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME
ｄｅｔａｉｌ ｄｅｓｃｒｉｐｔｉｏｎ ｓｅｅ ：
http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f21&sub=7BB3043221BA44E4BAF5760339280480
2017-02-16 23:27 GMT+08:00 Billy Liu <bi...@apache.org>:
> Could you reproduce this issue on the sample cube? That would help the dev
> team to identify the root cause quickly.
>
> 2017-02-16 20:30 GMT+08:00 xl l <yo...@gmail.com>:
>
> > HI, I am sure hbase is ok.
> > 而且只有这个sql抛异常，且能稳定复现。 sql稍微改一下，就正常。
> >
> > 从异常日志看， 首先抛出异常的是
> > Caused by: java.lang.NullPointerException
> > at com.google.common.base.Preconditions.checkNotNull(
> > Preconditions.java:191)
> > at
> > org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> > HBaseReadonlyStore.java:131)
> >
> > 对应于 代码：
> >
> > Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(i);
> > Cell cell = findCell(oneRow, hbaseColumn.getFirst(),
> > hbaseColumn.getSecond());
> > Preconditions.checkNotNull(cell);
> >
> > cell 啥时候 会为空？
> >
> >
> >
> >
> >
> > 2017-02-16 17:47 GMT+08:00 ShaoFeng Shi <sh...@apache.org>:
> >
> > > "java.net.SocketTimeoutException", did you check HBase's healthy
> status?
> > > Are all regions of the table "KYLIN_F2H6NPOLR7" online?
> > >
> > > 2017-02-16 16:10 GMT+08:00 xl l <yo...@gmail.com>:
> > >
> > > > see :
> > > >
> > > > http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f
> > > 21&sub=
> > > > 7BB3043221BA44E4BAF5760339280480
> > > >
> > > >
> > > > kylin 1.6 问题记录：
> > > > 版本：apache-kylin-1.6.0-hbase1.x-bin
> > > >
> > > >
> > > > select
> > > > cast(SUM(pv) as double) as pv,
> > > > cast( count(distinct user_id) as double) as user_id
> > > > from olap.olap_log_accs_page_di
> > > > inner join DIM.DIM_LOG_USER_LOCATION on
> > > > DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_
> > > > accs_page_di.location
> > > >
> > > > inner join DIM.DIM_PUBLIC_DATE_INFO on
> > > > DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_di."DATE"
> > > > where
> > > > DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
> > > > and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
> > > > and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
> > > > group by
> > > > DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_
> > > > USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_
> > > > LOCATION.user_region_name
> > > >
> > > > order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
> > > >
> > > > 上面这个ｓｑｌ执行ＯＫ，符合预期。
> > > > 但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
> > > >
> > > >
> > > > 错误信息如下所示：
> > > >
> > > >
> > > > 查看 kylin.log日志， 详细的异常信息 ：
> > > > http://note.youdao.com/noteshare?id=a1c257599774c4bccb0c6763923359
> > > d5&sub=
> > > > 11C6AA36AC894EDD9006DDAE17B16747
> > > > 2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
> > > > ipc.CoprocessorRpcChannel:58 : Call failed on IOException
> > > > java.net.SocketTimeoutException: callTimeout=60000,
> > callDuration=114625:
> > > > row ' ' on table 'KYLIN_F2H6NPOLR7' at
> > > > region=KYLIN_F2H6NPOLR7,,1487168742102.
> 433e266be82448c5380610e9e77046
> > > 58.,
> > > > hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
> > > > at
> > > > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(
> > > > RpcRetryingCaller.java:159)
> > > > at
> > > > org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.
> > callExecService(
> > > > RegionCoprocessorRpcChannel.java:95)
> > > > at
> > > > org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(
> > > > CoprocessorRpcChannel.java:56)
> > > > at
> > > > org.apache.kylin.storage.hbase.cube.v2.coprocessor.
> endpoint.generated.
> > > > CubeVisitProtos$CubeVisitService$Stub.visitCube(CubeVisitProtos.
> > > java:4178)
> > > >
> > > > Caused by: java.lang.NullPointerException
> > > > at com.google.common.base.Preconditions.checkNotNull(
> > > > Preconditions.java:191)
> > > > at
> > > > org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> > > > HBaseReadonlyStore.java:131)
> > > >
> > > >
> > > >
> > > > 根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
> > > >
> > > > 麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
> > > >
> > > >
> > > >
> > > >
> > > > 附上该sql其他情况下：
> > > > 如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select
> sum，count（distinct）
> > > 两个指标
> > > > 只出现一个 指标， 也不会报错。
> > > >
> > > > 补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
> > > >
> > > >
> > > > 附上 cube_desc 详细信息：
> > > >
> > > > { "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e", "last_modified":
> > > > 1487147245924, "version": "1.6.0", "name":
> > > > "olap_log_accs_page_di_cube_0215", "model_name":
> > > > "olap_log_accs_page_di_cube_0215", "description":
> > > > "olap_log_accs_page_di_cube_0215", "null_string": null,
> "dimensions":
> > [
> > > {
> > > > "name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> "YEAR",
> > > > "derived": null }, { "name": "QUARTER", "table":
> > > > "DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived": null
> },
> > {
> > > > "name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > > "MONTH_CN",
> > > > "derived": null }, { "name": "DATE", "table":
> > "DIM.DIM_PUBLIC_DATE_INFO",
> > > > "column": "DATE", "derived": null }, { "name": "PROVINCE", "table":
> > > > "DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME", "derived":
> null
> > > }, {
> > > > "name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column":
> > > "CITY_NAME",
> > > > "derived": null }, { "name": "USER_COUNTRY", "table":
> > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME",
> "derived":
> > > null
> > > > }, { "name": "USER_REGION", "table": "DIM.DIM_LOG_USER_LOCATION",
> > > "column":
> > > > "USER_REGION_NAME", "derived": null }, { "name": "USER_PROVINCE",
> > > "table":
> > > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME",
> "derived":
> > > > null }, { "name": "USER_CITY", "table": "DIM.DIM_LOG_USER_LOCATION",
> > > > "column": "USER_CITY_NAME", "derived": null }, { "name": "USER_TYPE",
> > > > "table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME", "derived":
> > null
> > > },
> > > > { "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN", "column":
> > > > "LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER", "table":
> > > > "DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME", "derived": null
> > },
> > > {
> > > > "name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
> > > > "BROWSER_NAME", "derived": null }, { "name": "APP_VERSION", "table":
> > > > "DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived": null
> }
> > ],
> > > > "measures": [ { "name": "_COUNT_", "function": { "expression":
> "COUNT",
> > > > "parameter": { "type": "constant", "value": "1", "next_parameter":
> null
> > > },
> > > > "returntype": "bigint" }, "dependent_measure_ref": null }, { "name":
> > > "PV",
> > > > "function": { "expression": "SUM", "parameter": { "type": "column",
> > > > "value": "PV", "next_parameter": null }, "returntype": "bigint" },
> > > > "dependent_measure_ref": null }, { "name": "UV", "function": {
> > > > "expression": "COUNT_DISTINCT", "parameter": { "type": "column",
> > "value":
> > > > "USER_ID", "next_parameter": null }, "returntype": "bitmap" },
> > > > "dependent_measure_ref": null }, { "name": "OUT_SESSION",
> "function": {
> > > > "expression": "SUM", "parameter": { "type": "column", "value":
> > > > "IS_OUT_SESSION", "next_parameter": null }, "returntype": "bigint" },
> > > > "dependent_measure_ref": null } ], "dictionaries": [], "rowkey": {
> > > > "rowkey_columns": [ { "column": "YEAR", "encoding": "dict",
> > "isShardBy":
> > > > false }, { "column": "QUARTER_CN", "encoding": "dict", "isShardBy":
> > false
> > > > }, { "column": "MONTH_CN", "encoding": "dict", "isShardBy": false },
> {
> > > > "column": "DATE", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > "PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > "CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > > "USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > "USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > > "USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> > > "column":
> > > > "USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > "TYPE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > > "LOGIN_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > > "REGISTER_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > > "BROWSER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > > "VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
> > > > "hbase_mapping": { "column_family": [ { "name": "F1", "columns": [ {
> > > > "qualifier": "M", "measure_refs": [ "_COUNT_", "PV", "OUT_SESSION" ]
> }
> > ]
> > > },
> > > > { "name": "F2", "columns": [ { "qualifier": "M", "measure_refs": [
> "UV"
> > > ] }
> > > > ] } ] }, "aggregation_groups": [ { "includes": [ "YEAR",
> "QUARTER_CN",
> > > > "MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME",
> "USER_COUNTRY_NAME",
> > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME",
> > "TYPE_NAME",
> > > > "LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME" ],
> > > > "select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN",
> > "MONTH_CN",
> > > > "DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [ "USER_COUNTRY_NAME",
> > > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
> > > > "mandatory_dims": [], "joint_dims": [] } } ], "signature":
> > > > "odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [], "status_need_notify":
> [
> > > > "ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start":
> > 1485907200000,
> > > > "partition_date_end": 3153600000000, "auto_merge_time_ranges": [
> > > 604800000,
> > > > 2419200000 ], "retention_range": 0, "engine_type": 2, "storage_type":
> > 2,
> > > > "override_kylin_properties": { "kylin.cube.aggrgroup.max.
> combination":
> > > > "16384" } }
> > > >
> > > >
> > > >
> > > >
> > > >
> > > >
> > > > --
> > > > 祝身体健康，万事如意 ^_^
> > > >
> > >
> > >
> > >
> > > --
> > > Best regards,
> > >
> > > Shaofeng Shi 史少锋
> > >
> >
> >
> >
> > --
> > 祝身体健康，万事如意 ^_^
> >
>
--
祝身体健康，万事如意 ^_^
Re: kylin sql query with Weird error
Posted by Billy Liu <bi...@apache.org>.
Could you reproduce this issue on the sample cube? That would help the dev
team to identify the root cause quickly.
2017-02-16 20:30 GMT+08:00 xl l <yo...@gmail.com>:
> HI, I am sure hbase is ok.
> 而且只有这个sql抛异常，且能稳定复现。 sql稍微改一下，就正常。
>
> 从异常日志看， 首先抛出异常的是
> Caused by: java.lang.NullPointerException
> at com.google.common.base.Preconditions.checkNotNull(
> Preconditions.java:191)
> at
> org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> HBaseReadonlyStore.java:131)
>
> 对应于 代码：
>
> Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(i);
> Cell cell = findCell(oneRow, hbaseColumn.getFirst(),
> hbaseColumn.getSecond());
> Preconditions.checkNotNull(cell);
>
> cell 啥时候 会为空？
>
>
>
>
>
> 2017-02-16 17:47 GMT+08:00 ShaoFeng Shi <sh...@apache.org>:
>
> > "java.net.SocketTimeoutException", did you check HBase's healthy status?
> > Are all regions of the table "KYLIN_F2H6NPOLR7" online?
> >
> > 2017-02-16 16:10 GMT+08:00 xl l <yo...@gmail.com>:
> >
> > > see :
> > >
> > > http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f
> > 21&sub=
> > > 7BB3043221BA44E4BAF5760339280480
> > >
> > >
> > > kylin 1.6 问题记录：
> > > 版本：apache-kylin-1.6.0-hbase1.x-bin
> > >
> > >
> > > select
> > > cast(SUM(pv) as double) as pv,
> > > cast( count(distinct user_id) as double) as user_id
> > > from olap.olap_log_accs_page_di
> > > inner join DIM.DIM_LOG_USER_LOCATION on
> > > DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_
> > > accs_page_di.location
> > >
> > > inner join DIM.DIM_PUBLIC_DATE_INFO on
> > > DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_di."DATE"
> > > where
> > > DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
> > > and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
> > > and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
> > > group by
> > > DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_
> > > USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_
> > > LOCATION.user_region_name
> > >
> > > order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
> > >
> > > 上面这个ｓｑｌ执行ＯＫ，符合预期。
> > > 但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
> > >
> > >
> > > 错误信息如下所示：
> > >
> > >
> > > 查看 kylin.log日志， 详细的异常信息 ：
> > > http://note.youdao.com/noteshare?id=a1c257599774c4bccb0c6763923359
> > d5&sub=
> > > 11C6AA36AC894EDD9006DDAE17B16747
> > > 2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
> > > ipc.CoprocessorRpcChannel:58 : Call failed on IOException
> > > java.net.SocketTimeoutException: callTimeout=60000,
> callDuration=114625:
> > > row ' ' on table 'KYLIN_F2H6NPOLR7' at
> > > region=KYLIN_F2H6NPOLR7,,1487168742102.433e266be82448c5380610e9e77046
> > 58.,
> > > hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
> > > at
> > > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(
> > > RpcRetryingCaller.java:159)
> > > at
> > > org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.
> callExecService(
> > > RegionCoprocessorRpcChannel.java:95)
> > > at
> > > org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(
> > > CoprocessorRpcChannel.java:56)
> > > at
> > > org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.
> > > CubeVisitProtos$CubeVisitService$Stub.visitCube(CubeVisitProtos.
> > java:4178)
> > >
> > > Caused by: java.lang.NullPointerException
> > > at com.google.common.base.Preconditions.checkNotNull(
> > > Preconditions.java:191)
> > > at
> > > org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> > > HBaseReadonlyStore.java:131)
> > >
> > >
> > >
> > > 根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
> > >
> > > 麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
> > >
> > >
> > >
> > >
> > > 附上该sql其他情况下：
> > > 如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select sum，count（distinct）
> > 两个指标
> > > 只出现一个 指标， 也不会报错。
> > >
> > > 补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
> > >
> > >
> > > 附上 cube_desc 详细信息：
> > >
> > > { "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e", "last_modified":
> > > 1487147245924, "version": "1.6.0", "name":
> > > "olap_log_accs_page_di_cube_0215", "model_name":
> > > "olap_log_accs_page_di_cube_0215", "description":
> > > "olap_log_accs_page_di_cube_0215", "null_string": null, "dimensions":
> [
> > {
> > > "name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column": "YEAR",
> > > "derived": null }, { "name": "QUARTER", "table":
> > > "DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived": null },
> {
> > > "name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> > "MONTH_CN",
> > > "derived": null }, { "name": "DATE", "table":
> "DIM.DIM_PUBLIC_DATE_INFO",
> > > "column": "DATE", "derived": null }, { "name": "PROVINCE", "table":
> > > "DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME", "derived": null
> > }, {
> > > "name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column":
> > "CITY_NAME",
> > > "derived": null }, { "name": "USER_COUNTRY", "table":
> > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME", "derived":
> > null
> > > }, { "name": "USER_REGION", "table": "DIM.DIM_LOG_USER_LOCATION",
> > "column":
> > > "USER_REGION_NAME", "derived": null }, { "name": "USER_PROVINCE",
> > "table":
> > > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME", "derived":
> > > null }, { "name": "USER_CITY", "table": "DIM.DIM_LOG_USER_LOCATION",
> > > "column": "USER_CITY_NAME", "derived": null }, { "name": "USER_TYPE",
> > > "table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME", "derived":
> null
> > },
> > > { "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN", "column":
> > > "LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER", "table":
> > > "DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME", "derived": null
> },
> > {
> > > "name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
> > > "BROWSER_NAME", "derived": null }, { "name": "APP_VERSION", "table":
> > > "DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived": null }
> ],
> > > "measures": [ { "name": "_COUNT_", "function": { "expression": "COUNT",
> > > "parameter": { "type": "constant", "value": "1", "next_parameter": null
> > },
> > > "returntype": "bigint" }, "dependent_measure_ref": null }, { "name":
> > "PV",
> > > "function": { "expression": "SUM", "parameter": { "type": "column",
> > > "value": "PV", "next_parameter": null }, "returntype": "bigint" },
> > > "dependent_measure_ref": null }, { "name": "UV", "function": {
> > > "expression": "COUNT_DISTINCT", "parameter": { "type": "column",
> "value":
> > > "USER_ID", "next_parameter": null }, "returntype": "bitmap" },
> > > "dependent_measure_ref": null }, { "name": "OUT_SESSION", "function": {
> > > "expression": "SUM", "parameter": { "type": "column", "value":
> > > "IS_OUT_SESSION", "next_parameter": null }, "returntype": "bigint" },
> > > "dependent_measure_ref": null } ], "dictionaries": [], "rowkey": {
> > > "rowkey_columns": [ { "column": "YEAR", "encoding": "dict",
> "isShardBy":
> > > false }, { "column": "QUARTER_CN", "encoding": "dict", "isShardBy":
> false
> > > }, { "column": "MONTH_CN", "encoding": "dict", "isShardBy": false }, {
> > > "column": "DATE", "encoding": "dict", "isShardBy": false }, { "column":
> > > "PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > "CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > "USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > "USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > > "USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> > "column":
> > > "USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > "TYPE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > "LOGIN_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > "REGISTER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > "BROWSER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > > "VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
> > > "hbase_mapping": { "column_family": [ { "name": "F1", "columns": [ {
> > > "qualifier": "M", "measure_refs": [ "_COUNT_", "PV", "OUT_SESSION" ] }
> ]
> > },
> > > { "name": "F2", "columns": [ { "qualifier": "M", "measure_refs": [ "UV"
> > ] }
> > > ] } ] }, "aggregation_groups": [ { "includes": [ "YEAR", "QUARTER_CN",
> > > "MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME", "USER_COUNTRY_NAME",
> > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME",
> "TYPE_NAME",
> > > "LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME" ],
> > > "select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN",
> "MONTH_CN",
> > > "DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [ "USER_COUNTRY_NAME",
> > > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
> > > "mandatory_dims": [], "joint_dims": [] } } ], "signature":
> > > "odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [], "status_need_notify": [
> > > "ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start":
> 1485907200000,
> > > "partition_date_end": 3153600000000, "auto_merge_time_ranges": [
> > 604800000,
> > > 2419200000 ], "retention_range": 0, "engine_type": 2, "storage_type":
> 2,
> > > "override_kylin_properties": { "kylin.cube.aggrgroup.max.combination":
> > > "16384" } }
> > >
> > >
> > >
> > >
> > >
> > >
> > > --
> > > 祝身体健康，万事如意 ^_^
> > >
> >
> >
> >
> > --
> > Best regards,
> >
> > Shaofeng Shi 史少锋
> >
>
>
>
> --
> 祝身体健康，万事如意 ^_^
>
Re: kylin sql query with Weird error
Posted by xl l <yo...@gmail.com>.
HI, I am sure hbase is ok.
而且只有这个sql抛异常，且能稳定复现。 sql稍微改一下，就正常。
从异常日志看， 首先抛出异常的是
Caused by: java.lang.NullPointerException
at com.google.common.base.Preconditions.checkNotNull(Preconditions.java:191)
at
org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(HBaseReadonlyStore.java:131)
对应于 代码：
Pair<byte[], byte[]> hbaseColumn = hbaseColumns.get(i);
Cell cell = findCell(oneRow, hbaseColumn.getFirst(), hbaseColumn.getSecond());
Preconditions.checkNotNull(cell);
cell 啥时候 会为空？
2017-02-16 17:47 GMT+08:00 ShaoFeng Shi <sh...@apache.org>:
> "java.net.SocketTimeoutException", did you check HBase's healthy status?
> Are all regions of the table "KYLIN_F2H6NPOLR7" online?
>
> 2017-02-16 16:10 GMT+08:00 xl l <yo...@gmail.com>:
>
> > see :
> >
> > http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f
> 21&sub=
> > 7BB3043221BA44E4BAF5760339280480
> >
> >
> > kylin 1.6 问题记录：
> > 版本：apache-kylin-1.6.0-hbase1.x-bin
> >
> >
> > select
> > cast(SUM(pv) as double) as pv,
> > cast( count(distinct user_id) as double) as user_id
> > from olap.olap_log_accs_page_di
> > inner join DIM.DIM_LOG_USER_LOCATION on
> > DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_
> > accs_page_di.location
> >
> > inner join DIM.DIM_PUBLIC_DATE_INFO on
> > DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_di."DATE"
> > where
> > DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
> > and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
> > and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
> > group by
> > DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_
> > USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_
> > LOCATION.user_region_name
> >
> > order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
> >
> > 上面这个ｓｑｌ执行ＯＫ，符合预期。
> > 但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
> >
> >
> > 错误信息如下所示：
> >
> >
> > 查看 kylin.log日志， 详细的异常信息 ：
> > http://note.youdao.com/noteshare?id=a1c257599774c4bccb0c6763923359
> d5&sub=
> > 11C6AA36AC894EDD9006DDAE17B16747
> > 2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
> > ipc.CoprocessorRpcChannel:58 : Call failed on IOException
> > java.net.SocketTimeoutException: callTimeout=60000, callDuration=114625:
> > row ' ' on table 'KYLIN_F2H6NPOLR7' at
> > region=KYLIN_F2H6NPOLR7,,1487168742102.433e266be82448c5380610e9e77046
> 58.,
> > hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
> > at
> > org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(
> > RpcRetryingCaller.java:159)
> > at
> > org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.callExecService(
> > RegionCoprocessorRpcChannel.java:95)
> > at
> > org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(
> > CoprocessorRpcChannel.java:56)
> > at
> > org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.
> > CubeVisitProtos$CubeVisitService$Stub.visitCube(CubeVisitProtos.
> java:4178)
> >
> > Caused by: java.lang.NullPointerException
> > at com.google.common.base.Preconditions.checkNotNull(
> > Preconditions.java:191)
> > at
> > org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> > HBaseReadonlyStore.java:131)
> >
> >
> >
> > 根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
> >
> > 麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
> >
> >
> >
> >
> > 附上该sql其他情况下：
> > 如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select sum，count（distinct）
> 两个指标
> > 只出现一个 指标， 也不会报错。
> >
> > 补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
> >
> >
> > 附上 cube_desc 详细信息：
> >
> > { "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e", "last_modified":
> > 1487147245924, "version": "1.6.0", "name":
> > "olap_log_accs_page_di_cube_0215", "model_name":
> > "olap_log_accs_page_di_cube_0215", "description":
> > "olap_log_accs_page_di_cube_0215", "null_string": null, "dimensions": [
> {
> > "name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column": "YEAR",
> > "derived": null }, { "name": "QUARTER", "table":
> > "DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived": null }, {
> > "name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column":
> "MONTH_CN",
> > "derived": null }, { "name": "DATE", "table": "DIM.DIM_PUBLIC_DATE_INFO",
> > "column": "DATE", "derived": null }, { "name": "PROVINCE", "table":
> > "DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME", "derived": null
> }, {
> > "name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column":
> "CITY_NAME",
> > "derived": null }, { "name": "USER_COUNTRY", "table":
> > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME", "derived":
> null
> > }, { "name": "USER_REGION", "table": "DIM.DIM_LOG_USER_LOCATION",
> "column":
> > "USER_REGION_NAME", "derived": null }, { "name": "USER_PROVINCE",
> "table":
> > "DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME", "derived":
> > null }, { "name": "USER_CITY", "table": "DIM.DIM_LOG_USER_LOCATION",
> > "column": "USER_CITY_NAME", "derived": null }, { "name": "USER_TYPE",
> > "table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME", "derived": null
> },
> > { "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN", "column":
> > "LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER", "table":
> > "DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME", "derived": null },
> {
> > "name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
> > "BROWSER_NAME", "derived": null }, { "name": "APP_VERSION", "table":
> > "DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived": null } ],
> > "measures": [ { "name": "_COUNT_", "function": { "expression": "COUNT",
> > "parameter": { "type": "constant", "value": "1", "next_parameter": null
> },
> > "returntype": "bigint" }, "dependent_measure_ref": null }, { "name":
> "PV",
> > "function": { "expression": "SUM", "parameter": { "type": "column",
> > "value": "PV", "next_parameter": null }, "returntype": "bigint" },
> > "dependent_measure_ref": null }, { "name": "UV", "function": {
> > "expression": "COUNT_DISTINCT", "parameter": { "type": "column", "value":
> > "USER_ID", "next_parameter": null }, "returntype": "bitmap" },
> > "dependent_measure_ref": null }, { "name": "OUT_SESSION", "function": {
> > "expression": "SUM", "parameter": { "type": "column", "value":
> > "IS_OUT_SESSION", "next_parameter": null }, "returntype": "bigint" },
> > "dependent_measure_ref": null } ], "dictionaries": [], "rowkey": {
> > "rowkey_columns": [ { "column": "YEAR", "encoding": "dict", "isShardBy":
> > false }, { "column": "QUARTER_CN", "encoding": "dict", "isShardBy": false
> > }, { "column": "MONTH_CN", "encoding": "dict", "isShardBy": false }, {
> > "column": "DATE", "encoding": "dict", "isShardBy": false }, { "column":
> > "PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > "USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, {
> "column":
> > "USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "TYPE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "LOGIN_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "REGISTER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "BROWSER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> > "VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
> > "hbase_mapping": { "column_family": [ { "name": "F1", "columns": [ {
> > "qualifier": "M", "measure_refs": [ "_COUNT_", "PV", "OUT_SESSION" ] } ]
> },
> > { "name": "F2", "columns": [ { "qualifier": "M", "measure_refs": [ "UV"
> ] }
> > ] } ] }, "aggregation_groups": [ { "includes": [ "YEAR", "QUARTER_CN",
> > "MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME", "USER_COUNTRY_NAME",
> > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME", "TYPE_NAME",
> > "LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME" ],
> > "select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN", "MONTH_CN",
> > "DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [ "USER_COUNTRY_NAME",
> > "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
> > "mandatory_dims": [], "joint_dims": [] } } ], "signature":
> > "odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [], "status_need_notify": [
> > "ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start": 1485907200000,
> > "partition_date_end": 3153600000000, "auto_merge_time_ranges": [
> 604800000,
> > 2419200000 ], "retention_range": 0, "engine_type": 2, "storage_type": 2,
> > "override_kylin_properties": { "kylin.cube.aggrgroup.max.combination":
> > "16384" } }
> >
> >
> >
> >
> >
> >
> > --
> > 祝身体健康，万事如意 ^_^
> >
>
>
>
> --
> Best regards,
>
> Shaofeng Shi 史少锋
>
--
祝身体健康，万事如意 ^_^
Re: kylin sql query with Weird error
Posted by ShaoFeng Shi <sh...@apache.org>.
"java.net.SocketTimeoutException", did you check HBase's healthy status?
Are all regions of the table "KYLIN_F2H6NPOLR7" online?
2017-02-16 16:10 GMT+08:00 xl l <yo...@gmail.com>:
> see :
>
> http://note.youdao.com/noteshare?id=df34c64dcf3cf801a9c085be0c3f5f21&sub=
> 7BB3043221BA44E4BAF5760339280480
>
>
> kylin 1.6 问题记录：
> 版本：apache-kylin-1.6.0-hbase1.x-bin
>
>
> select
> cast(SUM(pv) as double) as pv,
> cast( count(distinct user_id) as double) as user_id
> from olap.olap_log_accs_page_di
> inner join DIM.DIM_LOG_USER_LOCATION on
> DIM.DIM_LOG_USER_LOCATION.user_city_code=olap.olap_log_
> accs_page_di.location
>
> inner join DIM.DIM_PUBLIC_DATE_INFO on
> DIM.DIM_PUBLIC_DATE_INFO."DATE"=olap.olap_log_accs_page_di."DATE"
> where
> DIM.DIM_PUBLIC_DATE_INFO."DATE" >=20170117
> and DIM.DIM_PUBLIC_DATE_INFO."DATE" <=20170215
> and DIM.DIM_LOG_USER_LOCATION.user_region_name in ('华东')
> group by
> DIM.DIM_LOG_USER_LOCATION.user_country_name,DIM.DIM_LOG_
> USER_LOCATION.user_province_name,DIM.DIM_LOG_USER_
> LOCATION.user_region_name
>
> order by DIM.DIM_LOG_USER_LOCATION.user_province_name ASC
>
> 上面这个ｓｑｌ执行ＯＫ，符合预期。
> 但是 如果 仅仅把 in ('华东') 改成 in ('华东','华南') 则 sql执行就会报错。
>
>
> 错误信息如下所示：
>
>
> 查看 kylin.log日志， 详细的异常信息 ：
> http://note.youdao.com/noteshare?id=a1c257599774c4bccb0c6763923359d5&sub=
> 11C6AA36AC894EDD9006DDAE17B16747
> 2017-02-16 15:32:51,080 WARN [kylin-coproc--pool3-t5578]
> ipc.CoprocessorRpcChannel:58 : Call failed on IOException
> java.net.SocketTimeoutException: callTimeout=60000, callDuration=114625:
> row ' ' on table 'KYLIN_F2H6NPOLR7' at
> region=KYLIN_F2H6NPOLR7,,1487168742102.433e266be82448c5380610e9e7704658.,
> hostname=jx-db-hbase03.22lll.com,16020,1480406440673, seqNum=2
> at
> org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(
> RpcRetryingCaller.java:159)
> at
> org.apache.hadoop.hbase.ipc.RegionCoprocessorRpcChannel.callExecService(
> RegionCoprocessorRpcChannel.java:95)
> at
> org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel.callMethod(
> CoprocessorRpcChannel.java:56)
> at
> org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.
> CubeVisitProtos$CubeVisitService$Stub.visitCube(CubeVisitProtos.java:4178)
>
> Caused by: java.lang.NullPointerException
> at com.google.common.base.Preconditions.checkNotNull(
> Preconditions.java:191)
> at
> org.apache.kylin.storage.hbase.cube.v2.HBaseReadonlyStore$1$1.next(
> HBaseReadonlyStore.java:131)
>
>
>
> 根据kylin.log 异常，我的的判断定位，kylin源码中：HBaseReadonlyStore
>
> 麻烦帮忙排查，顺便咨询一下，这个源码中 cell啥情况下会为空？
>
>
>
>
> 附上该sql其他情况下：
> 如果 把sql 中 in ('华东') 改成 in ('华东','华南') 后 ，同时 select sum，count（distinct） 两个指标
> 只出现一个 指标， 也不会报错。
>
> 补充说明：在ｋｙｌｉｎ １.５.４.1 版本，该诡异现象 也存在。
>
>
> 附上 cube_desc 详细信息：
>
> { "uuid": "f30d538b-5345-4f77-b8e3-b20ebae8cb8e", "last_modified":
> 1487147245924, "version": "1.6.0", "name":
> "olap_log_accs_page_di_cube_0215", "model_name":
> "olap_log_accs_page_di_cube_0215", "description":
> "olap_log_accs_page_di_cube_0215", "null_string": null, "dimensions": [ {
> "name": "YEAR", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column": "YEAR",
> "derived": null }, { "name": "QUARTER", "table":
> "DIM.DIM_PUBLIC_DATE_INFO", "column": "QUARTER_CN", "derived": null }, {
> "name": "MONTH", "table": "DIM.DIM_PUBLIC_DATE_INFO", "column": "MONTH_CN",
> "derived": null }, { "name": "DATE", "table": "DIM.DIM_PUBLIC_DATE_INFO",
> "column": "DATE", "derived": null }, { "name": "PROVINCE", "table":
> "DIM.DIM_PUBLIC_CITY_INFO", "column": "PROVINCE_NAME", "derived": null }, {
> "name": "CITY", "table": "DIM.DIM_PUBLIC_CITY_INFO", "column": "CITY_NAME",
> "derived": null }, { "name": "USER_COUNTRY", "table":
> "DIM.DIM_LOG_USER_LOCATION", "column": "USER_COUNTRY_NAME", "derived": null
> }, { "name": "USER_REGION", "table": "DIM.DIM_LOG_USER_LOCATION", "column":
> "USER_REGION_NAME", "derived": null }, { "name": "USER_PROVINCE", "table":
> "DIM.DIM_LOG_USER_LOCATION", "column": "USER_PROVINCE_NAME", "derived":
> null }, { "name": "USER_CITY", "table": "DIM.DIM_LOG_USER_LOCATION",
> "column": "USER_CITY_NAME", "derived": null }, { "name": "USER_TYPE",
> "table": "DIM.DIM_LOG_USER_TYPE", "column": "TYPE_NAME", "derived": null },
> { "name": "IS_LOGIN", "table": "DIM.DIM_LOG_IS_LOGIN", "column":
> "LOGIN_NAME", "derived": null }, { "name": "IS_REGISTER", "table":
> "DIM.DIM_LOG_IS_REGISTER", "column": "REGISTER_NAME", "derived": null }, {
> "name": "BROWSER", "table": "DIM.DIM_LOG_BROWSER", "column":
> "BROWSER_NAME", "derived": null }, { "name": "APP_VERSION", "table":
> "DIM.DIM_LOG_APP_VERSION", "column": "VERSION_NAME", "derived": null } ],
> "measures": [ { "name": "_COUNT_", "function": { "expression": "COUNT",
> "parameter": { "type": "constant", "value": "1", "next_parameter": null },
> "returntype": "bigint" }, "dependent_measure_ref": null }, { "name": "PV",
> "function": { "expression": "SUM", "parameter": { "type": "column",
> "value": "PV", "next_parameter": null }, "returntype": "bigint" },
> "dependent_measure_ref": null }, { "name": "UV", "function": {
> "expression": "COUNT_DISTINCT", "parameter": { "type": "column", "value":
> "USER_ID", "next_parameter": null }, "returntype": "bitmap" },
> "dependent_measure_ref": null }, { "name": "OUT_SESSION", "function": {
> "expression": "SUM", "parameter": { "type": "column", "value":
> "IS_OUT_SESSION", "next_parameter": null }, "returntype": "bigint" },
> "dependent_measure_ref": null } ], "dictionaries": [], "rowkey": {
> "rowkey_columns": [ { "column": "YEAR", "encoding": "dict", "isShardBy":
> false }, { "column": "QUARTER_CN", "encoding": "dict", "isShardBy": false
> }, { "column": "MONTH_CN", "encoding": "dict", "isShardBy": false }, {
> "column": "DATE", "encoding": "dict", "isShardBy": false }, { "column":
> "PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "USER_COUNTRY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "USER_REGION_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "USER_PROVINCE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "USER_CITY_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "TYPE_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "LOGIN_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "REGISTER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "BROWSER_NAME", "encoding": "dict", "isShardBy": false }, { "column":
> "VERSION_NAME", "encoding": "dict", "isShardBy": false } ] },
> "hbase_mapping": { "column_family": [ { "name": "F1", "columns": [ {
> "qualifier": "M", "measure_refs": [ "_COUNT_", "PV", "OUT_SESSION" ] } ] },
> { "name": "F2", "columns": [ { "qualifier": "M", "measure_refs": [ "UV" ] }
> ] } ] }, "aggregation_groups": [ { "includes": [ "YEAR", "QUARTER_CN",
> "MONTH_CN", "DATE", "PROVINCE_NAME", "CITY_NAME", "USER_COUNTRY_NAME",
> "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME", "TYPE_NAME",
> "LOGIN_NAME", "REGISTER_NAME", "BROWSER_NAME", "VERSION_NAME" ],
> "select_rule": { "hierarchy_dims": [ [ "YEAR", "QUARTER_CN", "MONTH_CN",
> "DATE" ], [ "PROVINCE_NAME", "CITY_NAME" ], [ "USER_COUNTRY_NAME",
> "USER_REGION_NAME", "USER_PROVINCE_NAME", "USER_CITY_NAME" ] ],
> "mandatory_dims": [], "joint_dims": [] } } ], "signature":
> "odaKh4hY+Mb4uRggEJJanQ==", "notify_list": [], "status_need_notify": [
> "ERROR", "DISCARDED", "SUCCEED" ], "partition_date_start": 1485907200000,
> "partition_date_end": 3153600000000, "auto_merge_time_ranges": [ 604800000,
> 2419200000 ], "retention_range": 0, "engine_type": 2, "storage_type": 2,
> "override_kylin_properties": { "kylin.cube.aggrgroup.max.combination":
> "16384" } }
>
>
>
>
>
>
> --
> 祝身体健康，万事如意 ^_^
>
--
Best regards,
Shaofeng Shi 史少锋