You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/10/28 14:17:23 UTC
[1/2] kylin git commit: KYLIN-2137 Failed to run mr job when user put
a kafka jar in hive's lib folder
Repository: kylin
Updated Branches:
refs/heads/v1.6.0-rc1 46a894db6 -> a37ac46ac
KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e419681e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e419681e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e419681e
Branch: refs/heads/v1.6.0-rc1
Commit: e419681e8c9c6135af40dcdc2b9b3b367b3e2456
Parents: 46a894d
Author: shaofengshi <sh...@apache.org>
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Oct 28 22:11:33 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/StringUtil.java | 13 +++++++++
.../engine/mr/common/AbstractHadoopJob.java | 29 ++++++--------------
2 files changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
return result;
}
+ public static void appendWithSeparator(StringBuilder src, String append) {
+ if (src == null) {
+ throw new IllegalArgumentException();
+ }
+ if (src.length() > 0 && src.toString().endsWith(",") == false) {
+ src.append(",");
+ }
+
+ if (StringUtils.isBlank(append) == false) {
+ src.append(append);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e419681e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a5b2d2e..417d63d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
String filteredHive = filterKylinHiveDependency(kylinHiveDependency);
logger.info("Hive Dependencies After Filtered: " + filteredHive);
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(filteredHive);
+ StringUtil.appendWithSeparator(kylinDependency, filteredHive);
} else {
logger.info("No hive dependency jars set in the environment, will find them from jvm:");
try {
String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
- kylinDependency.append(hiveExecJarPath).append(",");
+
+ StringUtil.appendWithSeparator(kylinDependency, hiveExecJarPath);
logger.info("hive-exec jar file: " + hiveExecJarPath);
String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
- kylinDependency.append(hiveHCatJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, hiveHCatJarPath);
logger.info("hive-catalog jar file: " + hiveHCatJarPath);
String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
- kylinDependency.append(hiveMetaStoreJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, hiveMetaStoreJarPath);
logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath);
} catch (ClassNotFoundException e) {
logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for kafka dependencies
if (kylinKafkaDependency != null) {
kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
-
logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency);
-
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(kylinKafkaDependency);
+ StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency);
} else {
-
logger.info("No Kafka dependency jars set in the environment, will find them from jvm:");
-
try {
String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
- kylinDependency.append(kafkaClientJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath);
logger.info("kafka jar file: " + kafkaClientJarPath);
} catch (ClassNotFoundException e) {
@@ -251,12 +245,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
- if (!StringUtils.isBlank(mrLibDir)) {
- if (kylinDependency.length() > 0) {
- kylinDependency.append(",");
- }
- kylinDependency.append(mrLibDir);
- }
+ StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
setJobTmpJarsAndFiles(job, kylinDependency.toString());
[2/2] kylin git commit: KYLIN-2137 add check for
'kylin.job.mr.lib.dir'
Posted by sh...@apache.org.
KYLIN-2137 add check for 'kylin.job.mr.lib.dir'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a37ac46a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a37ac46a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a37ac46a
Branch: refs/heads/v1.6.0-rc1
Commit: a37ac46ace9fceda9780121d97348d6eb620f008
Parents: e419681
Author: shaofengshi <sh...@apache.org>
Authored: Fri Oct 28 22:17:04 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Oct 28 22:17:04 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a37ac46a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..a97a8c7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -245,7 +245,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
- StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+ if (!StringUtils.isBlank(mrLibDir)) {
+ File dirFileMRLIB = new File(mrLibDir);
+ if (dirFileMRLIB.exists()) {
+ StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+ } else {
+ logger.info("The directory '" + mrLibDir + "' for 'kylin.job.mr.lib.dir' does not exist!!!");
+ }
+ }
setJobTmpJarsAndFiles(job, kylinDependency.toString());