You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2016/04/18 08:34:56 UTC
[14/50] kylin git commit: KYLIN-1426: fix the bug: fail to locate
kylin.properties during cardinality calculation
KYLIN-1426: fix the bug: fail to locate kylin.properties during cardinality calculation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4656bd56
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4656bd56
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4656bd56
Branch: refs/heads/1.4-rc
Commit: 4656bd5643e09c67ebf81564a630d4163d61664f
Parents: 147986b
Author: yangzhong <ya...@ebay.com>
Authored: Thu Feb 18 09:44:18 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Feb 19 16:22:42 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/CubeService.java | 7 ++++---
.../source/hive/cardinality/HiveColumnCardinalityJob.java | 6 ++++++
2 files changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4656bd56/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 3f894ec..b543ef7 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -41,6 +41,7 @@ import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
+import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
@@ -499,10 +500,10 @@ public class CubeService extends BasicService {
String outPath = HiveColumnCardinalityJob.OUTPUT_PATH + "/" + tableName;
String param = "-table " + tableName + " -output " + outPath;
- HadoopShellExecutable step1 = new HadoopShellExecutable();
+ MapReduceExecutable step1 = new MapReduceExecutable();
- step1.setJobClass(HiveColumnCardinalityJob.class);
- step1.setJobParams(param);
+ step1.setMapReduceJobClass(HiveColumnCardinalityJob.class);
+ step1.setMapReduceParams(param);
job.addTask(step1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4656bd56/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
index 2b35755..2b12273 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
@@ -29,10 +29,13 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
/**
* This hadoop job will scan all rows of the hive table and then calculate the cardinality on each column.
@@ -94,6 +97,9 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
this.deletePath(job.getConfiguration(), output);
logger.info("Going to submit HiveColumnCardinalityJob for table '" + table + "'");
+
+ TableDesc tableDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(table);
+ attachKylinPropsAndMetadata(tableDesc, job.getConfiguration());
int result = waitForCompletion(job);
return result;