You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/30 10:44:03 UTC
[22/26] kylin git commit: KYLIN-2137 Failed to run mr job when user
put a kafka jar in hive's lib folder
KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8314e744
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8314e744
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8314e744
Branch: refs/heads/KYLIN-1971
Commit: 8314e744f40a4042f4652502c621cc30591b7fd1
Parents: a0cfaa1
Author: shaofengshi <sh...@apache.org>
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Oct 28 21:57:19 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/StringUtil.java | 13 +++++++++
.../engine/mr/common/AbstractHadoopJob.java | 29 ++++++--------------
2 files changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8314e744/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
return result;
}
+ public static void appendWithSeparator(StringBuilder src, String append) {
+ if (src == null) {
+ throw new IllegalArgumentException();
+ }
+ if (src.length() > 0 && src.toString().endsWith(",") == false) {
+ src.append(",");
+ }
+
+ if (StringUtils.isBlank(append) == false) {
+ src.append(append);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8314e744/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a5b2d2e..417d63d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
String filteredHive = filterKylinHiveDependency(kylinHiveDependency);
logger.info("Hive Dependencies After Filtered: " + filteredHive);
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(filteredHive);
+ StringUtil.appendWithSeparator(kylinDependency, filteredHive);
} else {
logger.info("No hive dependency jars set in the environment, will find them from jvm:");
try {
String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
- kylinDependency.append(hiveExecJarPath).append(",");
+
+ StringUtil.appendWithSeparator(kylinDependency, hiveExecJarPath);
logger.info("hive-exec jar file: " + hiveExecJarPath);
String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
- kylinDependency.append(hiveHCatJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, hiveHCatJarPath);
logger.info("hive-catalog jar file: " + hiveHCatJarPath);
String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
- kylinDependency.append(hiveMetaStoreJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, hiveMetaStoreJarPath);
logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath);
} catch (ClassNotFoundException e) {
logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for kafka dependencies
if (kylinKafkaDependency != null) {
kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
-
logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency);
-
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(kylinKafkaDependency);
+ StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency);
} else {
-
logger.info("No Kafka dependency jars set in the environment, will find them from jvm:");
-
try {
String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
- kylinDependency.append(kafkaClientJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath);
logger.info("kafka jar file: " + kafkaClientJarPath);
} catch (ClassNotFoundException e) {
@@ -251,12 +245,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
- if (!StringUtils.isBlank(mrLibDir)) {
- if (kylinDependency.length() > 0) {
- kylinDependency.append(",");
- }
- kylinDependency.append(mrLibDir);
- }
+ StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
setJobTmpJarsAndFiles(job, kylinDependency.toString());