You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/06 09:30:42 UTC
[06/49] kylin git commit: AbstractHadoopJob: set hive dependency
filter as a config entry
AbstractHadoopJob: set hive dependency filter as a config entry
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca976f31
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca976f31
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca976f31
Branch: refs/heads/KYLIN-1971
Commit: ca976f31910de0e4b69533028dfc259e6f6d6f78
Parents: 02f8550
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Oct 31 17:03:42 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Oct 31 17:07:52 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 10 +++++++---
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 9 +++------
2 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca976f31/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4f294fa..acc4eb1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -22,9 +22,9 @@ import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.SortedSet;
-import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -430,7 +430,7 @@ abstract public class KylinConfigBase implements Serializable {
public int getCubeAlgorithmAutoMapperLimit() {
return Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500"));
}
-
+
@Deprecated
public int getCubeAggrGroupMaxSize() {
return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12"));
@@ -500,7 +500,7 @@ abstract public class KylinConfigBase implements Serializable {
public boolean getBadQueryPersistentEnabled() {
return Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable", "true"));
}
-
+
public String[] getQueryTransformers() {
return getOptionalStringArray("kylin.query.transformers", new String[0]);
}
@@ -629,6 +629,10 @@ abstract public class KylinConfigBase implements Serializable {
return this.getOptional("kylin.job.hive.database.for.intermediatetable", "default");
}
+ public String getHiveDependencyFilterList() {
+ return this.getOptional("kylin.job.dependency.filterlist", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar");
+ }
+
public String getKylinOwner() {
return this.getOptional("kylin.owner", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca976f31/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 72f4437..77791ce 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -96,8 +96,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
- private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar";
-
protected static void runJob(Tool job, String[] args) {
try {
int exitCode = ToolRunner.run(job, args);
@@ -200,8 +198,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
kylinHiveDependency = kylinHiveDependency.replace(":", ",");
logger.info("Hive Dependencies Before Filtered: " + kylinHiveDependency);
- //String filteredHive = filterKylinHiveDependency(kylinHiveDependency);
- String filteredHive = kylinHiveDependency;
+ String filteredHive = filterKylinHiveDependency(kylinHiveDependency, kylinConf);
logger.info("Hive Dependencies After Filtered: " + filteredHive);
StringUtil.appendWithSeparator(kylinDependency, filteredHive);
@@ -259,13 +256,13 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
}
- private String filterKylinHiveDependency(String kylinHiveDependency) {
+ private String filterKylinHiveDependency(String kylinHiveDependency, KylinConfig config) {
if (StringUtils.isBlank(kylinHiveDependency))
return "";
StringBuilder jarList = new StringBuilder();
- Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS);
+ Pattern hivePattern = Pattern.compile(config.getHiveDependencyFilterList());
Matcher matcher = hivePattern.matcher(kylinHiveDependency);
while (matcher.find()) {