You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2018/01/24 09:44:35 UTC
kylin git commit: KYLIN-3020 org.apache.hadoop.util.ToolRunner is not
threadsafe and misused in kylin HadoopShellExecutable
Repository: kylin
Updated Branches:
refs/heads/master 4044c820d -> 47bec8595
KYLIN-3020 org.apache.hadoop.util.ToolRunner is not threadsafe and misused in kylin HadoopShellExecutable
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47bec859
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47bec859
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47bec859
Branch: refs/heads/master
Commit: 47bec8595c8d64abb58f911d96e6bf0d20b76eeb
Parents: 4044c82
Author: Ma,Gang <ga...@ebay.com>
Authored: Mon Nov 13 15:20:27 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Jan 24 17:44:21 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/common/HadoopShellExecutable.java | 4 ++--
.../java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java | 3 ++-
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/47bec859/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java
index 10a6e8c..f6609c2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java
@@ -22,8 +22,8 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.reflect.Constructor;
-import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.exception.HadoopShellException;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -60,7 +60,7 @@ public class HadoopShellExecutable extends AbstractExecutable {
int result;
StringBuilder log = new StringBuilder();
try {
- result = ToolRunner.run(job, args);
+ result = MRUtil.runMRJob(job, args);
} catch (Exception ex) {
logger.error("error execute " + this.toString(), ex);
StringWriter stringWriter = new StringWriter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/47bec859/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index d1caccb..f0b77aa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -74,7 +75,7 @@ public class BulkLoadJob extends AbstractHadoopJob {
newArgs[1] = tableName;
logger.debug("Start to run LoadIncrementalHFiles");
- int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs);
+ int ret = MRUtil.runMRJob(new LoadIncrementalHFiles(conf), newArgs);
logger.debug("End to run LoadIncrementalHFiles");
return ret;
}