You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/03/23 07:59:11 UTC
[22/50] [abbrv] kylin git commit: KYLIN-1507 find hive dependency jar
on some platform like CDH
KYLIN-1507 find hive dependency jar on some platform like CDH
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dba135f4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dba135f4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dba135f4
Branch: refs/heads/master
Commit: dba135f4c8d834fce53f0d3490a77af6e204e24e
Parents: 30cccbd
Author: Yang Li <li...@apache.org>
Authored: Sun Mar 20 20:13:08 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Mar 20 20:13:08 2016 +0800
----------------------------------------------------------------------
.../engine/mr/common/AbstractHadoopJob.java | 68 ++++++++++----------
1 file changed, 35 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dba135f4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index fe60ca8..04b591f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -78,7 +78,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube name. For exmaple, flat_item_cube").create(BatchConstants.ARG_CUBE_NAME);
protected static final Option OPTION_CUBING_JOB_ID = OptionBuilder.withArgName(BatchConstants.ARG_CUBING_JOB_ID).hasArg().isRequired(false).withDescription("ID of cubing job executable").create(BatchConstants.ARG_CUBING_JOB_ID);
protected static final Option OPTION_II_NAME = OptionBuilder.withArgName(BatchConstants.ARG_II_NAME).hasArg().isRequired(true).withDescription("II name. For exmaple, some_ii").create(BatchConstants.ARG_II_NAME);
- protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName( BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create( BatchConstants.ARG_SEGMENT_NAME);
+ protected static final Option OPTION_SEGMENT_NAME = OptionBuilder.withArgName(BatchConstants.ARG_SEGMENT_NAME).hasArg().isRequired(true).withDescription("Cube segment name").create(BatchConstants.ARG_SEGMENT_NAME);
protected static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_INPUT).hasArg().isRequired(true).withDescription("Input path").create(BatchConstants.ARG_INPUT);
protected static final Option OPTION_INPUT_FORMAT = OptionBuilder.withArgName(BatchConstants.ARG_INPUT_FORMAT).hasArg().isRequired(false).withDescription("Input format").create(BatchConstants.ARG_INPUT_FORMAT);
protected static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_OUTPUT).hasArg().isRequired(true).withDescription("Output path").create(BatchConstants.ARG_OUTPUT);
@@ -90,6 +90,22 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Option OPTION_STATISTICS_OUTPUT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_OUTPUT).hasArg().isRequired(false).withDescription("Statistics output").create(BatchConstants.ARG_STATS_OUTPUT);
protected static final Option OPTION_STATISTICS_SAMPLING_PERCENT = OptionBuilder.withArgName(BatchConstants.ARG_STATS_SAMPLING_PERCENT).hasArg().isRequired(false).withDescription("Statistics sampling percentage").create(BatchConstants.ARG_STATS_SAMPLING_PERCENT);
+ private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
+
+ private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar";
+
+ protected static void runJob(Tool job, String[] args) {
+ try {
+ int exitCode = ToolRunner.run(job, args);
+ System.exit(exitCode);
+ } catch (Exception e) {
+ e.printStackTrace(System.err);
+ System.exit(5);
+ }
+ }
+
+ // ============================================================================
+
protected String name;
protected boolean isAsync = false;
protected OptionsHelper optionsHelper = new OptionsHelper();
@@ -137,38 +153,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return retVal;
}
- protected static void runJob(Tool job, String[] args) {
- try {
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- } catch (Exception e) {
- e.printStackTrace(System.err);
- System.exit(5);
- }
- }
-
- private static final String KYLIN_HIVE_DEPENDENCY_JARS = "[^,]*hive-exec[0-9.-]+\\.jar|[^,]*hive-metastore[0-9.-]+\\.jar|[^,]*hive-hcatalog-core[0-9.-]+\\.jar";
-
- String filterKylinHiveDependency(String kylinHiveDependency) {
- if (StringUtils.isBlank(kylinHiveDependency))
- return "";
-
- StringBuilder jarList = new StringBuilder();
-
- Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS);
- Matcher matcher = hivePattern.matcher(kylinHiveDependency);
-
- while (matcher.find()) {
- if (jarList.length() > 0)
- jarList.append(",");
- jarList.append(matcher.group());
- }
-
- return jarList.toString();
- }
-
- private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
-
protected void setJobClasspath(Job job) {
String jarPath = KylinConfig.getInstanceFromEnv().getKylinJobJarPath();
File jarFile = new File(jarPath);
@@ -254,6 +238,24 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
setJobTmpJarsAndFiles(job, kylinDependency.toString());
}
+ private String filterKylinHiveDependency(String kylinHiveDependency) {
+ if (StringUtils.isBlank(kylinHiveDependency))
+ return "";
+
+ StringBuilder jarList = new StringBuilder();
+
+ Pattern hivePattern = Pattern.compile(KYLIN_HIVE_DEPENDENCY_JARS);
+ Matcher matcher = hivePattern.matcher(kylinHiveDependency);
+
+ while (matcher.find()) {
+ if (jarList.length() > 0)
+ jarList.append(",");
+ jarList.append(matcher.group());
+ }
+
+ return jarList.toString();
+ }
+
private void setJobTmpJarsAndFiles(Job job, String kylinDependency) {
if (StringUtils.isBlank(kylinDependency))
return;