You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@kylin.apache.org by 装甲车 <ti...@qq.com> on 2019/03/12 09:52:05 UTC
回复： please help me
I was install kylin and hive in the VM(three machines) , the VM memory has 2048M, is that enough memory?
------------------ 原始邮件 ------------------
发件人: "ShaoFeng Shi"<sh...@apache.org>;
发送时间: 2019年3月12日(星期二) 下午2:12
收件人: "user"<us...@kylin.apache.org>;
主题: Re: please help me
OS command error exit with return code: 137
I remember the 137 actually represents 9, which is OOM. Try to increase the memory allocation for Hive.
Best regards,
Shaofeng Shi 史少锋
Apache Kylin PMC
Email: shaofengshi@apache.org
Apache Kylin FAQ: https://kylin.apache.org/docs/gettingstarted/faq.html
Join Kylin user mail group: user-subscribe@kylin.apache.org
Join Kylin dev mail group: dev-subscribe@kylin.apache.org
装甲车 <ti...@qq.com> 于2019年3月12日周二 上午10:57写道：
when i build cube , have an error ,
the error is :
java.io.IOException: OS command error exit with return code: 137, error message:
Logging initialized using configuration in jar:file:/usr/local/hive/lib/hive-common-2.3.4.jar!/hive-log4j2.properties Async: true
OK
Time taken: 9.515 seconds
OK
Time taken: 0.429 seconds
OK
Time taken: 2.318 seconds
OK
Time taken: 0.769 seconds
No Stats for default@kylin_sales, Columns: part_dt, ops_user_id, lstg_format_name, price, ops_region, lstg_site_id, trans_id, leaf_categ_id, buyer_id, seller_id
No Stats for default@kylin_cal_dt, Columns: cal_dt
No Stats for default@kylin_category_groupings, Columns: meta_categ_name, site_id, categ_lvl2_name, leaf_categ_id, categ_lvl3_name
No Stats for default@kylin_account, Columns: account_id, account_country, account_buyer_level
No Stats for default@kylin_account, Columns: account_seller_level, account_id, account_country
No Stats for default@kylin_country, Columns: country, name
No Stats for default@kylin_country, Columns: country, name
The command is:
hive -e "USE default;
DROP TABLE IF EXISTS kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d;
CREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d
(
KYLIN_SALES_TRANS_ID bigint
,KYLIN_SALES_PART_DT date
,KYLIN_SALES_LEAF_CATEG_ID bigint
,KYLIN_SALES_LSTG_SITE_ID int
,KYLIN_CATEGORY_GROUPINGS_META_CATEG_NAME string
,KYLIN_CATEGORY_GROUPINGS_CATEG_LVL2_NAME string
,KYLIN_CATEGORY_GROUPINGS_CATEG_LVL3_NAME string
,KYLIN_SALES_LSTG_FORMAT_NAME string
,KYLIN_SALES_SELLER_ID bigint
,KYLIN_SALES_BUYER_ID bigint
,BUYER_ACCOUNT_ACCOUNT_BUYER_LEVEL int
,SELLER_ACCOUNT_ACCOUNT_SELLER_LEVEL int
,BUYER_ACCOUNT_ACCOUNT_COUNTRY string
,SELLER_ACCOUNT_ACCOUNT_COUNTRY string
,BUYER_COUNTRY_NAME string
,SELLER_COUNTRY_NAME string
,KYLIN_SALES_OPS_USER_ID string
,KYLIN_SALES_OPS_REGION string
,KYLIN_SALES_PRICE decimal(19,4)
)
STORED AS SEQUENCEFILE
LOCATION 'hdfs://master:9000/kylin/kylin_metadata/kylin-8afd9685-40cc-5000-5e36-a1d66bc8cba9/kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d';
ALTER TABLE kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d SET TBLPROPERTIES('auto.purge'='true');
INSERT OVERWRITE TABLE \`kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d\` SELECT
\`KYLIN_SALES\`.\`TRANS_ID\` as \`KYLIN_SALES_TRANS_ID\`
,\`KYLIN_SALES\`.\`PART_DT\` as \`KYLIN_SALES_PART_DT\`
,\`KYLIN_SALES\`.\`LEAF_CATEG_ID\` as \`KYLIN_SALES_LEAF_CATEG_ID\`
,\`KYLIN_SALES\`.\`LSTG_SITE_ID\` as \`KYLIN_SALES_LSTG_SITE_ID\`
,\`KYLIN_CATEGORY_GROUPINGS\`.\`META_CATEG_NAME\` as \`KYLIN_CATEGORY_GROUPINGS_META_CATEG_NAME\`
,\`KYLIN_CATEGORY_GROUPINGS\`.\`CATEG_LVL2_NAME\` as \`KYLIN_CATEGORY_GROUPINGS_CATEG_LVL2_NAME\`
,\`KYLIN_CATEGORY_GROUPINGS\`.\`CATEG_LVL3_NAME\` as \`KYLIN_CATEGORY_GROUPINGS_CATEG_LVL3_NAME\`
,\`KYLIN_SALES\`.\`LSTG_FORMAT_NAME\` as \`KYLIN_SALES_LSTG_FORMAT_NAME\`
,\`KYLIN_SALES\`.\`SELLER_ID\` as \`KYLIN_SALES_SELLER_ID\`
,\`KYLIN_SALES\`.\`BUYER_ID\` as \`KYLIN_SALES_BUYER_ID\`
,\`BUYER_ACCOUNT\`.\`ACCOUNT_BUYER_LEVEL\` as \`BUYER_ACCOUNT_ACCOUNT_BUYER_LEVEL\`
,\`SELLER_ACCOUNT\`.\`ACCOUNT_SELLER_LEVEL\` as \`SELLER_ACCOUNT_ACCOUNT_SELLER_LEVEL\`
,\`BUYER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` as \`BUYER_ACCOUNT_ACCOUNT_COUNTRY\`
,\`SELLER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` as \`SELLER_ACCOUNT_ACCOUNT_COUNTRY\`
,\`BUYER_COUNTRY\`.\`NAME\` as \`BUYER_COUNTRY_NAME\`
,\`SELLER_COUNTRY\`.\`NAME\` as \`SELLER_COUNTRY_NAME\`
,\`KYLIN_SALES\`.\`OPS_USER_ID\` as \`KYLIN_SALES_OPS_USER_ID\`
,\`KYLIN_SALES\`.\`OPS_REGION\` as \`KYLIN_SALES_OPS_REGION\`
,\`KYLIN_SALES\`.\`PRICE\` as \`KYLIN_SALES_PRICE\`
FROM \`DEFAULT\`.\`KYLIN_SALES\` as \`KYLIN_SALES\`
INNER JOIN \`DEFAULT\`.\`KYLIN_CAL_DT\` as \`KYLIN_CAL_DT\`
ON \`KYLIN_SALES\`.\`PART_DT\` = \`KYLIN_CAL_DT\`.\`CAL_DT\`
INNER JOIN \`DEFAULT\`.\`KYLIN_CATEGORY_GROUPINGS\` as \`KYLIN_CATEGORY_GROUPINGS\`
ON \`KYLIN_SALES\`.\`LEAF_CATEG_ID\` = \`KYLIN_CATEGORY_GROUPINGS\`.\`LEAF_CATEG_ID\` AND \`KYLIN_SALES\`.\`LSTG_SITE_ID\` = \`KYLIN_CATEGORY_GROUPINGS\`.\`SITE_ID\`
INNER JOIN \`DEFAULT\`.\`KYLIN_ACCOUNT\` as \`BUYER_ACCOUNT\`
ON \`KYLIN_SALES\`.\`BUYER_ID\` = \`BUYER_ACCOUNT\`.\`ACCOUNT_ID\`
INNER JOIN \`DEFAULT\`.\`KYLIN_ACCOUNT\` as \`SELLER_ACCOUNT\`
ON \`KYLIN_SALES\`.\`SELLER_ID\` = \`SELLER_ACCOUNT\`.\`ACCOUNT_ID\`
INNER JOIN \`DEFAULT\`.\`KYLIN_COUNTRY\` as \`BUYER_COUNTRY\`
ON \`BUYER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` = \`BUYER_COUNTRY\`.\`COUNTRY\`
INNER JOIN \`DEFAULT\`.\`KYLIN_COUNTRY\` as \`SELLER_COUNTRY\`
ON \`SELLER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` = \`SELLER_COUNTRY\`.\`COUNTRY\`
WHERE 1=1 AND (\`KYLIN_SALES\`.\`PART_DT\` >= '2012-01-01' AND \`KYLIN_SALES\`.\`PART_DT\` < '2019-03-01')
;
" --hiveconf hive.merge.mapredfiles=false --hiveconf hive.auto.convert.join=true --hiveconf dfs.replication=2 --hiveconf hive.exec.compress.output=true --hiveconf hive.auto.convert.join.noconditionaltask=true --hiveconf mapreduce.job.split.metainfo.maxsize=-1 --hiveconf hive.merge.mapfiles=false --hiveconf hive.auto.convert.join.noconditionaltask.size=100000000 --hiveconf hive.stats.autogather=true
at org.apache.kylin.common.util.CliCommandExecutor.execute(CliCommandExecutor.java:96)
at org.apache.kylin.source.hive.CreateFlatHiveTableStep.createFlatHiveTable(CreateFlatHiveTableStep.java:62)
at org.apache.kylin.source.hive.CreateFlatHiveTableStep.doWork(CreateFlatHiveTableStep.java:99)
at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:165)
at org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:70)
at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:165)
at org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:114)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Re: please help me
Posted by JiaTao Tao <ta...@gmail.com>.
Hi
It's not enough, see(http://kylin.apache.org/docs/install/index.html):
We recommend you to try out Kylin or develop it using the integrated
> sandbox, such as HDP sandbox, and make sure it has at least 10 GB of
> memory. When configuring a sandbox, we recommend that you use the Bridged
> Adapter model instead of the NAT model.
>
装甲车 <ti...@qq.com> 于2019年3月12日周二 上午9:59写道：
> I was install kylin and hive in the VM(three machines) , the VM memory has
> 2048M, is that enough memory?
>
>
> ------------------ 原始邮件 ------------------
> *发件人:* "ShaoFeng Shi"<sh...@apache.org>;
> *发送时间:* 2019年3月12日(星期二) 下午2:12
> *收件人:* "user"<us...@kylin.apache.org>;
> *主题:* Re: please help me
>
> OS command error exit with return code: 137
>
> I remember the 137 actually represents 9, which is OOM. Try to increase
> the memory allocation for Hive.
>
> Best regards,
>
> Shaofeng Shi 史少锋
> Apache Kylin PMC
> Email: shaofengshi@apache.org
>
> Apache Kylin FAQ: https://kylin.apache.org/docs/gettingstarted/faq.html
> Join Kylin user mail group: user-subscribe@kylin.apache.org
> Join Kylin dev mail group: dev-subscribe@kylin.apache.org
>
>
>
>
> 装甲车 <ti...@qq.com> 于2019年3月12日周二 上午10:57写道：
>
>> when i build cube , have an error ,
>>
>> the error is :
>>
>>
>>
>> java.io.IOException: OS command error exit with return code: 137,
>> error message:
>> Logging initialized using configuration in
>> jar:file:/usr/local/hive/lib/hive-common-2.3.4.jar!/hive-log4j2.properties
>> Async: true
>> OK
>> Time taken: 9.515 seconds
>> OK
>> Time taken: 0.429 seconds
>> OK
>> Time taken: 2.318 seconds
>> OK
>> Time taken: 0.769 seconds
>> No Stats for default@kylin_sales, Columns: part_dt, ops_user_id,
>> lstg_format_name, price, ops_region, lstg_site_id, trans_id, leaf_categ_id,
>> buyer_id, seller_id
>> No Stats for default@kylin_cal_dt, Columns: cal_dt
>> No Stats for default@kylin_category_groupings, Columns: meta_categ_name,
>> site_id, categ_lvl2_name, leaf_categ_id, categ_lvl3_name
>> No Stats for default@kylin_account, Columns: account_id,
>> account_country, account_buyer_level
>> No Stats for default@kylin_account, Columns: account_seller_level,
>> account_id, account_country
>> No Stats for default@kylin_country, Columns: country, name
>> No Stats for default@kylin_country, Columns: country, name
>> The command is:
>> hive -e "USE default;
>>
>> DROP TABLE IF EXISTS
>> kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d;
>> CREATE EXTERNAL TABLE IF NOT EXISTS
>> kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d
>> (
>> KYLIN_SALES_TRANS_ID bigint
>> ,KYLIN_SALES_PART_DT date
>> ,KYLIN_SALES_LEAF_CATEG_ID bigint
>> ,KYLIN_SALES_LSTG_SITE_ID int
>> ,KYLIN_CATEGORY_GROUPINGS_META_CATEG_NAME string
>> ,KYLIN_CATEGORY_GROUPINGS_CATEG_LVL2_NAME string
>> ,KYLIN_CATEGORY_GROUPINGS_CATEG_LVL3_NAME string
>> ,KYLIN_SALES_LSTG_FORMAT_NAME string
>> ,KYLIN_SALES_SELLER_ID bigint
>> ,KYLIN_SALES_BUYER_ID bigint
>> ,BUYER_ACCOUNT_ACCOUNT_BUYER_LEVEL int
>> ,SELLER_ACCOUNT_ACCOUNT_SELLER_LEVEL int
>> ,BUYER_ACCOUNT_ACCOUNT_COUNTRY string
>> ,SELLER_ACCOUNT_ACCOUNT_COUNTRY string
>> ,BUYER_COUNTRY_NAME string
>> ,SELLER_COUNTRY_NAME string
>> ,KYLIN_SALES_OPS_USER_ID string
>> ,KYLIN_SALES_OPS_REGION string
>> ,KYLIN_SALES_PRICE decimal(19,4)
>> )
>> STORED AS SEQUENCEFILE
>> LOCATION
>> 'hdfs://master:9000/kylin/kylin_metadata/kylin-8afd9685-40cc-5000-5e36-a1d66bc8cba9/kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d';
>> ALTER TABLE
>> kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d
>> SET TBLPROPERTIES('auto.purge'='true');
>> INSERT OVERWRITE TABLE
>> \`kylin_intermediate_kylin_sales_cube_a41ef3ef_8166_177d_d5ee_886c268c7e4d\`
>> SELECT
>> \`KYLIN_SALES\`.\`TRANS_ID\` as \`KYLIN_SALES_TRANS_ID\`
>> ,\`KYLIN_SALES\`.\`PART_DT\` as \`KYLIN_SALES_PART_DT\`
>> ,\`KYLIN_SALES\`.\`LEAF_CATEG_ID\` as \`KYLIN_SALES_LEAF_CATEG_ID\`
>> ,\`KYLIN_SALES\`.\`LSTG_SITE_ID\` as \`KYLIN_SALES_LSTG_SITE_ID\`
>> ,\`KYLIN_CATEGORY_GROUPINGS\`.\`META_CATEG_NAME\` as
>> \`KYLIN_CATEGORY_GROUPINGS_META_CATEG_NAME\`
>> ,\`KYLIN_CATEGORY_GROUPINGS\`.\`CATEG_LVL2_NAME\` as
>> \`KYLIN_CATEGORY_GROUPINGS_CATEG_LVL2_NAME\`
>> ,\`KYLIN_CATEGORY_GROUPINGS\`.\`CATEG_LVL3_NAME\` as
>> \`KYLIN_CATEGORY_GROUPINGS_CATEG_LVL3_NAME\`
>> ,\`KYLIN_SALES\`.\`LSTG_FORMAT_NAME\` as \`KYLIN_SALES_LSTG_FORMAT_NAME\`
>> ,\`KYLIN_SALES\`.\`SELLER_ID\` as \`KYLIN_SALES_SELLER_ID\`
>> ,\`KYLIN_SALES\`.\`BUYER_ID\` as \`KYLIN_SALES_BUYER_ID\`
>> ,\`BUYER_ACCOUNT\`.\`ACCOUNT_BUYER_LEVEL\` as
>> \`BUYER_ACCOUNT_ACCOUNT_BUYER_LEVEL\`
>> ,\`SELLER_ACCOUNT\`.\`ACCOUNT_SELLER_LEVEL\` as
>> \`SELLER_ACCOUNT_ACCOUNT_SELLER_LEVEL\`
>> ,\`BUYER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` as
>> \`BUYER_ACCOUNT_ACCOUNT_COUNTRY\`
>> ,\`SELLER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` as
>> \`SELLER_ACCOUNT_ACCOUNT_COUNTRY\`
>> ,\`BUYER_COUNTRY\`.\`NAME\` as \`BUYER_COUNTRY_NAME\`
>> ,\`SELLER_COUNTRY\`.\`NAME\` as \`SELLER_COUNTRY_NAME\`
>> ,\`KYLIN_SALES\`.\`OPS_USER_ID\` as \`KYLIN_SALES_OPS_USER_ID\`
>> ,\`KYLIN_SALES\`.\`OPS_REGION\` as \`KYLIN_SALES_OPS_REGION\`
>> ,\`KYLIN_SALES\`.\`PRICE\` as \`KYLIN_SALES_PRICE\`
>> FROM \`DEFAULT\`.\`KYLIN_SALES\` as \`KYLIN_SALES\`
>> INNER JOIN \`DEFAULT\`.\`KYLIN_CAL_DT\` as \`KYLIN_CAL_DT\`
>> ON \`KYLIN_SALES\`.\`PART_DT\` = \`KYLIN_CAL_DT\`.\`CAL_DT\`
>> INNER JOIN \`DEFAULT\`.\`KYLIN_CATEGORY_GROUPINGS\` as
>> \`KYLIN_CATEGORY_GROUPINGS\`
>> ON \`KYLIN_SALES\`.\`LEAF_CATEG_ID\` =
>> \`KYLIN_CATEGORY_GROUPINGS\`.\`LEAF_CATEG_ID\` AND
>> \`KYLIN_SALES\`.\`LSTG_SITE_ID\` = \`KYLIN_CATEGORY_GROUPINGS\`.\`SITE_ID\`
>> INNER JOIN \`DEFAULT\`.\`KYLIN_ACCOUNT\` as \`BUYER_ACCOUNT\`
>> ON \`KYLIN_SALES\`.\`BUYER_ID\` = \`BUYER_ACCOUNT\`.\`ACCOUNT_ID\`
>> INNER JOIN \`DEFAULT\`.\`KYLIN_ACCOUNT\` as \`SELLER_ACCOUNT\`
>> ON \`KYLIN_SALES\`.\`SELLER_ID\` = \`SELLER_ACCOUNT\`.\`ACCOUNT_ID\`
>> INNER JOIN \`DEFAULT\`.\`KYLIN_COUNTRY\` as \`BUYER_COUNTRY\`
>> ON \`BUYER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` = \`BUYER_COUNTRY\`.\`COUNTRY\`
>> INNER JOIN \`DEFAULT\`.\`KYLIN_COUNTRY\` as \`SELLER_COUNTRY\`
>> ON \`SELLER_ACCOUNT\`.\`ACCOUNT_COUNTRY\` = \`SELLER_COUNTRY\`.\`COUNTRY\`
>> WHERE 1=1 AND (\`KYLIN_SALES\`.\`PART_DT\` >= '2012-01-01' AND
>> \`KYLIN_SALES\`.\`PART_DT\` < '2019-03-01')
>> ;
>>
>> " --hiveconf hive.merge.mapredfiles=false --hiveconf
>> hive.auto.convert.join=true --hiveconf dfs.replication=2 --hiveconf
>> hive.exec.compress.output=true --hiveconf
>> hive.auto.convert.join.noconditionaltask=true --hiveconf
>> mapreduce.job.split.metainfo.maxsize=-1 --hiveconf
>> hive.merge.mapfiles=false --hiveconf
>> hive.auto.convert.join.noconditionaltask.size=100000000 --hiveconf
>> hive.stats.autogather=true
>> at
>> org.apache.kylin.common.util.CliCommandExecutor.execute(CliCommandExecutor.java:96)
>> at
>> org.apache.kylin.source.hive.CreateFlatHiveTableStep.createFlatHiveTable(CreateFlatHiveTableStep.java:62)
>> at
>> org.apache.kylin.source.hive.CreateFlatHiveTableStep.doWork(CreateFlatHiveTableStep.java:99)
>> at
>> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:165)
>> at
>> org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:70)
>> at
>> org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:165)
>> at
>> org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:114)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
>> at java.lang.Thread.run(Thread.java:748)
>>
>
--
Regards!
Aron Tao