You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/08/14 05:21:26 UTC
[21/50] [abbrv] incubator-kylin git commit: KYLIN-921 Revert
KYLIN-921 Revert
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e2bbc467
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e2bbc467
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e2bbc467
Branch: refs/heads/0.7
Commit: e2bbc467ee89318068864fef09bee5088dc7cc43
Parents: aea42d7
Author: shaofengshi <sh...@apache.org>
Authored: Mon Aug 10 12:37:56 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Aug 10 12:37:56 2015 +0800
----------------------------------------------------------------------
.../kylin/job/constant/BatchConstants.java | 3 --
.../job/hadoop/cube/FactDistinctColumnsJob.java | 40 ++----------------
.../hadoop/cube/FactDistinctColumnsMapper.java | 43 ++++++++++++++++----
.../hadoop/cube/FactDistinctColumnsReducer.java | 39 ++++--------------
4 files changed, 45 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
index 7333753..e6b16a9 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
@@ -32,9 +32,6 @@ public interface BatchConstants {
public static final String CFG_II_NAME = "ii.name";
public static final String CFG_II_SEGMENT_NAME = "ii.segment.name";
- public static final String CFG_FACT_DICT_COLUMN_NAMES = "fact.dict.column.names";
- public static final String CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES = "fact.dict.column.rowkey.indexes";
-
public static final String INPUT_DELIM = "input.delim";
public static final String OUTPUT_PATH = "output.path";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index e701092..0a0b5a4 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -18,43 +18,34 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.IOException;
-import java.util.List;
-
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
+import java.io.IOException;
/**
* @author yangli9
*/
public class FactDistinctColumnsJob extends AbstractHadoopJob {
protected static final Logger log = LoggerFactory.getLogger(FactDistinctColumnsJob.class);
- private static final Joiner joiner = Joiner.on(",");
@Override
public int run(String[] args) throws Exception {
@@ -78,7 +69,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
// add metadata to distributed cache
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cubeInstance = cubeMgr.getCube(cubeName);
- CubeDesc cubeDesc = cubeInstance.getDescriptor();
jobConf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
System.out.println("Starting: " + job.getJobName());
@@ -89,31 +79,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
setupReducer(output);
// CubeSegment seg = cubeMgr.getCube(cubeName).getTheOnlySegment();
- attachKylinPropsAndMetadata(cubeInstance, jobConf);
-
- // set names and row key indexes of fact table's dictionary columns in Configuration,
- // so that mapper & reducer task can use them
- List<String> factDictColNames = Lists.newArrayList();
- List<Integer> factDictColRowKeyIndexes = Lists.newArrayList();
-
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, Cuboid.getBaseCuboidId(cubeDesc));
- List<TblColRef> columns = baseCuboid.getColumns();
-
- RowKeyDesc rowkey = cubeDesc.getRowkey();
- DictionaryManager dictMgr = DictionaryManager.getInstance(KylinConfig.getInstanceFromEnv());
- for (int i = 0; i < columns.size(); i++) {
- TblColRef col = columns.get(i);
- if (!rowkey.isUseDictionary(col))
- continue;
-
- String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), rowkey.getDictionary(col), col, null)[0];
- if (cubeDesc.getModel().isFactTable(scanTable)) {
- factDictColNames.add(col.getName());
- factDictColRowKeyIndexes.add(i);
- }
- }
- jobConf.set(BatchConstants.CFG_FACT_DICT_COLUMN_NAMES, joiner.join(factDictColNames));
- jobConf.set(BatchConstants.CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES, joiner.join(factDictColRowKeyIndexes));
+ attachKylinPropsAndMetadata(cubeInstance, job.getConfiguration());
return waitForCompletion(job);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
index b2ec6c6..518c7c4 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
@@ -19,6 +19,8 @@
package org.apache.kylin.job.hadoop.cube;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.ShortWritable;
@@ -32,18 +34,26 @@ import org.apache.kylin.common.mr.KylinMapper;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.RowKeyDesc;
+import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.job.hadoop.hive.CubeJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
/**
* @author yangli9
*/
public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRecord, ShortWritable, Text> {
+ private String cubeName;
+ private CubeInstance cube;
+ private CubeDesc cubeDesc;
+ private int[] factDictCols;
+
private CubeJoinedFlatTableDesc intermediateTableDesc;
- private int[] factDictColRowKeyIndexes;
private ShortWritable outputKey = new ShortWritable();
private Text outputValue = new Text();
@@ -58,16 +68,31 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
Configuration conf = context.getConfiguration();
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
- String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
- CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
- CubeDesc cubeDesc = cube.getDescriptor();
+ cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
+ cube = CubeManager.getInstance(config).getCube(cubeName);
+ cubeDesc = cube.getDescriptor();
intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
- String[] rowKeyIndexes = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES).split(",");
- factDictColRowKeyIndexes = new int[rowKeyIndexes.length];
- for (int i = 0; i < rowKeyIndexes.length; i++) {
- factDictColRowKeyIndexes[i] = Integer.parseInt(rowKeyIndexes[i]);
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ List<TblColRef> columns = baseCuboid.getColumns();
+
+ ArrayList<Integer> factDictCols = new ArrayList<Integer>();
+ RowKeyDesc rowkey = cubeDesc.getRowkey();
+ DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+ for (int i = 0; i < columns.size(); i++) {
+ TblColRef col = columns.get(i);
+ if (rowkey.isUseDictionary(col) == false)
+ continue;
+
+ String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
+ if (cubeDesc.getModel().isFactTable(scanTable)) {
+ factDictCols.add(i);
+ }
}
+ this.factDictCols = new int[factDictCols.size()];
+ for (int i = 0; i < factDictCols.size(); i++)
+ this.factDictCols[i] = factDictCols.get(i);
schema = HCatInputFormat.getTableSchema(context.getConfiguration());
}
@@ -79,7 +104,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
int[] flatTableIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
HCatFieldSchema fieldSchema = null;
- for (int i : factDictColRowKeyIndexes) {
+ for (int i : factDictCols) {
outputKey.set((short) i);
fieldSchema = schema.get(flatTableIndexes[i]);
Object fieldValue = record.get(fieldSchema.getName(), schema);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index 411f6d9..b161bfa 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -18,13 +18,6 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -43,19 +36,18 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
/**
* @author yangli9
*/
public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text, NullWritable, Text> {
- private static Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
- private String outputPath;
- private FileSystem fs;
private List<TblColRef> columnList = new ArrayList<TblColRef>();
- private Set<String> toBeExtractedColumns = new HashSet<String>();
@Override
protected void setup(Context context) throws IOException {
@@ -70,18 +62,11 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
columnList = baseCuboid.getColumns();
-
- outputPath = conf.get(BatchConstants.OUTPUT_PATH);
- fs = FileSystem.get(conf);
-
- String[] factDictColNames = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_NAMES).split(",");
- toBeExtractedColumns.addAll(Arrays.asList(factDictColNames));
}
@Override
public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
TblColRef col = columnList.get(key.get());
- toBeExtractedColumns.remove(col.getName());
HashSet<ByteArray> set = new HashSet<ByteArray>();
for (Text textValue : values) {
@@ -89,6 +74,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
set.add(value);
}
+ Configuration conf = context.getConfiguration();
+ FileSystem fs = FileSystem.get(conf);
+ String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
FSDataOutputStream out = fs.create(new Path(outputPath, col.getName()));
try {
@@ -102,15 +90,4 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
}
- @Override
- protected void cleanup(Context context) throws IOException, InterruptedException {
- // KYLIN-921 create empty file for dimension with all nulls
- for (String colName : toBeExtractedColumns) {
- logger.info("create empty file for column " + colName);
- boolean success = fs.createNewFile(new Path(outputPath, colName));
- if (!success) {
- throw new IOException("failed to create empty file for column " + colName);
- }
- }
- }
}