You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/05/18 08:15:13 UTC
kylin git commit: KYLIN-1706 Override MR job conf by kylin properties
Repository: kylin
Updated Branches:
refs/heads/master 185b34ff6 -> f0fe457e3
KYLIN-1706 Override MR job conf by kylin properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0fe457e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0fe457e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0fe457e
Branch: refs/heads/master
Commit: f0fe457e387235107921ee58f5a7e23a53938d02
Parents: 185b34f
Author: Li Yang <li...@apache.org>
Authored: Wed May 18 16:14:54 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed May 18 16:14:54 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 18 ++
.../apache/kylin/common/KylinConfigTest.java | 52 ++++++
.../kylin/cube/CubeSpecificConfigTest.java | 130 +++++++--------
.../engine/mr/common/AbstractHadoopJob.java | 15 +-
.../test_case_data/localmeta/kylin.properties | 164 ++++++++++---------
5 files changed, 227 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 14dda82..f4b359f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -22,6 +22,8 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.SortedSet;
import java.util.regex.Matcher;
@@ -33,6 +35,7 @@ import org.apache.kylin.common.util.CliCommandExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@SuppressWarnings("serial")
@@ -85,6 +88,17 @@ abstract public class KylinConfigBase implements Serializable {
protected Properties getAllProperties() {
return properties;
}
+
+ final protected Map<String, String> getPropertiesByPrefix(String prefix) {
+ Map<String, String> result = Maps.newLinkedHashMap();
+ for (Entry<Object, Object> entry : getAllProperties().entrySet()) {
+ String key = (String) entry.getKey();
+ if (key.startsWith(prefix)) {
+ result.put(key.substring(prefix.length()), (String) entry.getValue());
+ }
+ }
+ return result;
+ }
final protected String[] getOptionalStringArray(String prop, String[] dft) {
final String property = getOptional(prop);
@@ -225,6 +239,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.job.mr.lib.dir", "");
}
+ public Map<String, String> getMRConfigOverride() {
+ return getPropertiesByPrefix("kylin.job.mr.config.override.");
+ }
+
public String getKylinSparkJobJarPath() {
final String jobJar = getOptional("kylin.job.jar.spark");
if (StringUtils.isNotEmpty(jobJar)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
new file mode 100644
index 0000000..9ce1425
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common;
+
+import static org.junit.Assert.*;
+
+import java.util.Map;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class KylinConfigTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testMRConfigOverride() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ Map<String, String> override = config.getMRConfigOverride();
+ assertEquals(2, override.size());
+ assertEquals("test1", override.get("test1"));
+ assertEquals("test2", override.get("test2"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
index 00cc66c..51d4e54 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java
@@ -1,70 +1,60 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CubeSpecificConfigTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void test() {
- KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
- CubeDesc cubeDesc = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb");
- verifyOverride(baseConfig, cubeDesc.getConfig());
- }
-
- @Test
- public void test2() {
- KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
- CubeInstance cube = CubeManager.getInstance(baseConfig).getCube("ssb");
- verifyOverride(baseConfig, cube.getConfig());
- }
-
- private void verifyOverride(KylinConfig base, KylinConfig override) {
- assertEquals("gzip", base.getHbaseDefaultCompressionCodec());
- assertEquals("lz4", override.getHbaseDefaultCompressionCodec());
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import static org.junit.Assert.*;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CubeSpecificConfigTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void test() {
+ KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+ CubeDesc cubeDesc = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb");
+ verifyOverride(baseConfig, cubeDesc.getConfig());
+ }
+
+ @Test
+ public void test2() {
+ KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
+ CubeInstance cube = CubeManager.getInstance(baseConfig).getCube("ssb");
+ verifyOverride(baseConfig, cube.getConfig());
+ }
+
+ private void verifyOverride(KylinConfig base, KylinConfig override) {
+ assertEquals("gzip", base.getHbaseDefaultCompressionCodec());
+ assertEquals("lz4", override.getHbaseDefaultCompressionCodec());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index e4f253f..737ef0e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -30,6 +30,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -154,7 +155,9 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
protected void setJobClasspath(Job job) {
- String jarPath = KylinConfig.getInstanceFromEnv().getKylinJobJarPath();
+ KylinConfig kylinConf = KylinConfig.getInstanceFromEnv();
+
+ String jarPath = kylinConf.getKylinJobJarPath();
File jarFile = new File(jarPath);
if (jarFile.exists()) {
job.setJar(jarPath);
@@ -223,7 +226,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
// for KylinJobMRLibDir
- String mrLibDir = KylinConfig.getInstanceFromEnv().getKylinJobMRLibDir();
+ String mrLibDir = kylinConf.getKylinJobMRLibDir();
if (!StringUtils.isBlank(mrLibDir)) {
File dirFileMRLIB = new File(mrLibDir);
if (dirFileMRLIB.exists()) {
@@ -236,6 +239,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
setJobTmpJarsAndFiles(job, kylinDependency.toString());
+
+ overrideJobConfig(job.getConfiguration(), kylinConf.getMRConfigOverride());
+ }
+
+ private void overrideJobConfig(Configuration jobConf, Map<String, String> override) {
+ for (Entry<String, String> entry : override.entrySet()) {
+ jobConf.set(entry.getKey(), entry.getValue());
+ }
}
private String filterKylinHiveDependency(String kylinHiveDependency) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0fe457e/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 051a312..9ac7625 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -1,80 +1,84 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# optional information for the owner of kylin platform, it can be your team's email
-# currently it will be attached to each kylin's htable attribute
-kylin.owner=whoami@kylin.apache.org
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-#kylin.rest.servers=localhost:7070
-
-# The metadata store in hbase
-kylin.metadata.url=
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=false
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=10
-
-#default compression codec for htable,snappy,lzo,gzip,lz4
-kylin.hbase.default.compression.codec=gzip
-
-
-kylin.security.profile=testing
-## Config for Restful APP ##
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
-
-
-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# optional information for the owner of kylin platform, it can be your team's email
+# currently it will be attached to each kylin's htable attribute
+kylin.owner=whoami@kylin.apache.org
+
+# List of web servers in use, this enables one web server instance to sync up with other servers.
+#kylin.rest.servers=localhost:7070
+
+# The metadata store in hbase
+kylin.metadata.url=
+
+# The storage for final cube file in hbase
+kylin.storage.url=hbase
+
+# Temp folder in hdfs, make sure user has the right access to the hdfs directory
+kylin.hdfs.working.dir=/kylin
+
+kylin.job.mapreduce.default.reduce.input.mb=500
+
+# If true, job engine will not assume that hadoop CLI reside on the same server as it self
+# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
+kylin.job.run.as.remote.cmd=false
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.hostname=
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.username=
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.password=
+
+# Used by test cases to prepare synthetic data for sample cube
+kylin.job.remote.cli.working.dir=/tmp/kylin
+
+# Max count of concurrent jobs running
+kylin.job.concurrent.max.limit=10
+
+# Time interval to check hadoop job status
+kylin.job.yarn.app.rest.check.interval.seconds=10
+
+#default compression codec for htable,snappy,lzo,gzip,lz4
+kylin.hbase.default.compression.codec=gzip
+
+# test for hadoop job config override
+kylin.job.mr.config.override.test1=test1
+kylin.job.mr.config.override.test2=test2
+
+
+kylin.security.profile=testing
+## Config for Restful APP ##
+# database connection settings:
+ldap.server=
+ldap.username=
+ldap.password=
+ldap.user.searchBase=
+ldap.user.searchPattern=
+ldap.user.groupSearchBase=
+ldap.service.searchBase=OU=
+ldap.service.searchPattern=
+ldap.service.groupSearchBase=
+acl.adminRole=
+acl.defaultRole=
+ganglia.group=
+ganglia.port=8664
+
+
+