You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/02 02:57:46 UTC
incubator-kylin git commit: add attachKylinPropsAndMetadata in
IIDistinctColumnsJob
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-1112 46ed2432f -> 431c20b67
add attachKylinPropsAndMetadata in IIDistinctColumnsJob
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/431c20b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/431c20b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/431c20b6
Branch: refs/heads/KYLIN-1112
Commit: 431c20b67930a3d555e80c865566e77a4ec23b7e
Parents: 46ed243
Author: shaofengshi <sh...@apache.org>
Authored: Mon Nov 2 09:57:21 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Nov 2 09:57:21 2015 +0800
----------------------------------------------------------------------
.../engine/mr/common/AbstractHadoopJob.java | 22 ++++++++++++++++++++
.../mr/invertedindex/IIDistinctColumnsJob.java | 3 +++
.../mr/invertedindex/InvertedIndexJob.java | 21 +------------------
3 files changed, 26 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/431c20b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 1455664..c460d62 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -58,6 +58,8 @@ import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.job.exception.JobException;
@@ -322,6 +324,26 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
attachKylinPropsAndMetadata(dumpList, conf);
}
+ protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
+ MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+
+ // write II / model_desc / II_desc / dict / table
+ ArrayList<String> dumpList = new ArrayList<String>();
+ dumpList.add(ii.getResourcePath());
+ dumpList.add(ii.getDescriptor().getModel().getResourcePath());
+ dumpList.add(ii.getDescriptor().getResourcePath());
+
+ for (String tableName : ii.getDescriptor().getModel().getAllTables()) {
+ TableDesc table = metaMgr.getTableDesc(tableName);
+ dumpList.add(table.getResourcePath());
+ }
+ for (IISegment segment : ii.getSegments()) {
+ dumpList.addAll(segment.getDictionaryPaths());
+ }
+
+ attachKylinPropsAndMetadata(dumpList, conf);
+ }
+
protected void attachKylinPropsAndMetadata(ArrayList<String> dumpList, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
FileUtils.forceDelete(tmp); // we need a directory, so delete the file first
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/431c20b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIDistinctColumnsJob.java
index b691dc1..e9534f2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIDistinctColumnsJob.java
@@ -21,6 +21,7 @@ package org.apache.kylin.engine.mr.invertedindex;
import java.io.IOException;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.ShortWritable;
@@ -81,6 +82,8 @@ public class IIDistinctColumnsJob extends AbstractHadoopJob {
setupMapper(ii.getFirstSegment());
setupReducer(output);
+ Configuration conf = job.getConfiguration();
+ attachKylinPropsAndMetadata(ii, conf);
return waitForCompletion(job);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/431c20b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
index cff7219..bcae524 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java
@@ -108,26 +108,7 @@ public class InvertedIndexJob extends AbstractHadoopJob {
conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName());
}
- protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- // write II / model_desc / II_desc / dict / table
- ArrayList<String> dumpList = new ArrayList<String>();
- dumpList.add(ii.getResourcePath());
- dumpList.add(ii.getDescriptor().getModel().getResourcePath());
- dumpList.add(ii.getDescriptor().getResourcePath());
-
- for (String tableName : ii.getDescriptor().getModel().getAllTables()) {
- TableDesc table = metaMgr.getTableDesc(tableName);
- dumpList.add(table.getResourcePath());
- }
- for (IISegment segment : ii.getSegments()) {
- dumpList.addAll(segment.getDictionaryPaths());
- }
-
- attachKylinPropsAndMetadata(dumpList, conf);
- }
-
+
private void setupMapper(IISegment segment) throws IOException {
// String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);