You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/11/21 11:49:24 UTC
[kylin] branch master updated: Update BatchMergeJobBuilder2.java
(#855)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 2e571e6 Update BatchMergeJobBuilder2.java (#855)
2e571e6 is described below
commit 2e571e6b106d09b30a2260e061e848fe89a488fc
Author: yz490700477 <40...@users.noreply.github.com>
AuthorDate: Thu Nov 21 19:49:17 2019 +0800
Update BatchMergeJobBuilder2.java (#855)
"Merge Cuboid Dictionary" step should load MapReduce config from kylin_job_conf.xml instead of kylin_job_conf_inmem.xml
---
build/conf/kylin_job_conf_cube_merge.xml | 104 +++++++++++++++++++++
.../apache/kylin/job/engine/JobEngineConfig.java | 1 +
.../kylin/engine/mr/BatchMergeJobBuilder2.java | 8 +-
.../kylin/engine/mr/StreamingCubingJobBuilder.java | 2 +-
4 files changed, 110 insertions(+), 5 deletions(-)
diff --git a/build/conf/kylin_job_conf_cube_merge.xml b/build/conf/kylin_job_conf_cube_merge.xml
new file mode 100644
index 0000000..79365ad
--- /dev/null
+++ b/build/conf/kylin_job_conf_cube_merge.xml
@@ -0,0 +1,104 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>mapreduce.job.split.metainfo.maxsize</name>
+ <value>-1</value>
+ <description>The maximum permissible size of the split metainfo file.
+ The JobTracker won't attempt to read split metainfo files bigger than
+ the configured value. No limits if set to -1.
+ </description>
+ </property>
+
+ <property>
+ <name>mapreduce.map.output.compress</name>
+ <value>true</value>
+ <description>Compress map outputs</description>
+ </property>
+
+ <!--
+ The default map outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
+ <property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+ <description>The compression codec to use for map outputs
+ </description>
+ </property>
+ -->
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>true</value>
+ <description>Compress the output of a MapReduce job</description>
+ </property>
+ <!--
+ The default job outputs compress codec is org.apache.hadoop.io.compress.DefaultCodec,
+ if SnappyCodec is supported, org.apache.hadoop.io.compress.SnappyCodec could be used.
+ -->
+ <!--
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+ <description>The compression codec to use for job outputs
+ </description>
+ </property>
+ -->
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+ <description>The compression type to use for job outputs</description>
+ </property>
+
+ <property>
+ <name>mapreduce.job.max.split.locations</name>
+ <value>2000</value>
+ <description>No description</description>
+ </property>
+
+ <property>
+ <name>dfs.replication</name>
+ <value>2</value>
+ <description>Block replication</description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.timeout</name>
+ <value>7200000</value>
+ <description>Set task timeout to 1 hour</description>
+ </property>
+
+ <!--Additional config for cube merge job, giving more memory -->
+ <property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>3072</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Xmx2700m -XX:OnOutOfMemoryError='kill -9 %p'</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>200</value>
+ <description></description>
+ </property>
+</configuration>
diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index c0892ed..2694e25 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -36,6 +36,7 @@ public class JobEngineConfig {
public static final String HADOOP_JOB_CONF_FILENAME = "kylin_job_conf";
public static final String DEFAULT_JOB_CONF_SUFFIX = "";
public static final String IN_MEM_JOB_CONF_SUFFIX = "inmem";
+ public static final String CUBE_MERGE_JOB_CONF_SUFFIX = "cube_merge";
private static File getJobConfig(String fileName) {
String path = System.getProperty(KylinConfig.KYLIN_CONF);
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
index fc10c5c..a26500a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchMergeJobBuilder2.java
@@ -6,15 +6,15 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.engine.mr;
@@ -79,7 +79,7 @@ public class BatchMergeJobBuilder2 extends JobBuilderSupport {
MapReduceExecutable mergeDictionaryStep = new MapReduceExecutable();
mergeDictionaryStep.setName(ExecutableConstants.STEP_NAME_MERGE_DICTIONARY);
StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);
+ appendMapReduceParameters(cmd, JobEngineConfig.CUBE_MERGE_JOB_CONF_SUFFIX);
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getCubeInstance().getName());
appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingJobBuilder.java
index 28b9987..1f85b04 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingJobBuilder.java
@@ -107,7 +107,7 @@ public class StreamingCubingJobBuilder extends JobBuilderSupport {
mergeDict.setName(ExecutableConstants.STEP_NAME_STREAMING_CREATE_DICTIONARY);
StringBuilder cmd = new StringBuilder();
- appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);
+ appendMapReduceParameters(cmd, JobEngineConfig.CUBE_MERGE_JOB_CONF_SUFFIX);
appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
ExecutableConstants.STEP_NAME_STREAMING_CREATE_DICTIONARY);
appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, streamingStoragePath);