You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/08/14 05:21:26 UTC

[21/50] [abbrv] incubator-kylin git commit: KYLIN-921 Revert

KYLIN-921 Revert

Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e2bbc467
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e2bbc467
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e2bbc467

Branch: refs/heads/0.7
Commit: e2bbc467ee89318068864fef09bee5088dc7cc43
Parents: aea42d7
Author: shaofengshi <sh...@apache.org>
Authored: Mon Aug 10 12:37:56 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Aug 10 12:37:56 2015 +0800

----------------------------------------------------------------------
 .../kylin/job/constant/BatchConstants.java      |  3 --
 .../job/hadoop/cube/FactDistinctColumnsJob.java | 40 ++----------------
 .../hadoop/cube/FactDistinctColumnsMapper.java  | 43 ++++++++++++++++----
 .../hadoop/cube/FactDistinctColumnsReducer.java | 39 ++++--------------
 4 files changed, 45 insertions(+), 80 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
index 7333753..e6b16a9 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
@@ -32,9 +32,6 @@ public interface BatchConstants {
     public static final String CFG_II_NAME = "ii.name";
     public static final String CFG_II_SEGMENT_NAME = "ii.segment.name";
 
-    public static final String CFG_FACT_DICT_COLUMN_NAMES = "fact.dict.column.names";
-    public static final String CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES = "fact.dict.column.rowkey.indexes";
-
     public static final String INPUT_DELIM = "input.delim";
 
     public static final String OUTPUT_PATH = "output.path";

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index e701092..0a0b5a4 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -18,43 +18,34 @@
 
 package org.apache.kylin.job.hadoop.cube;
 
-import java.io.IOException;
-import java.util.List;
-
 import org.apache.commons.cli.Options;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.ShortWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.HadoopUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.job.constant.BatchConstants;
 import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
+import java.io.IOException;
 
 /**
  * @author yangli9
  */
 public class FactDistinctColumnsJob extends AbstractHadoopJob {
     protected static final Logger log = LoggerFactory.getLogger(FactDistinctColumnsJob.class);
-    private static final Joiner joiner = Joiner.on(",");
 
     @Override
     public int run(String[] args) throws Exception {
@@ -78,7 +69,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
             // add metadata to distributed cache
             CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
             CubeInstance cubeInstance = cubeMgr.getCube(cubeName);
-            CubeDesc cubeDesc = cubeInstance.getDescriptor();
 
             jobConf.set(BatchConstants.CFG_CUBE_NAME, cubeName);
             System.out.println("Starting: " + job.getJobName());
@@ -89,31 +79,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
             setupReducer(output);
 
             // CubeSegment seg = cubeMgr.getCube(cubeName).getTheOnlySegment();
-            attachKylinPropsAndMetadata(cubeInstance, jobConf);
-
-            // set names and row key indexes of fact table's dictionary columns in Configuration,
-            // so that mapper & reducer task can use them
-            List<String> factDictColNames = Lists.newArrayList();
-            List<Integer> factDictColRowKeyIndexes = Lists.newArrayList();
-
-            Cuboid baseCuboid = Cuboid.findById(cubeDesc, Cuboid.getBaseCuboidId(cubeDesc));
-            List<TblColRef> columns = baseCuboid.getColumns();
-
-            RowKeyDesc rowkey = cubeDesc.getRowkey();
-            DictionaryManager dictMgr = DictionaryManager.getInstance(KylinConfig.getInstanceFromEnv());
-            for (int i = 0; i < columns.size(); i++) {
-                TblColRef col = columns.get(i);
-                if (!rowkey.isUseDictionary(col))
-                    continue;
-
-                String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), rowkey.getDictionary(col), col, null)[0];
-                if (cubeDesc.getModel().isFactTable(scanTable)) {
-                    factDictColNames.add(col.getName());
-                    factDictColRowKeyIndexes.add(i);
-                }
-            }
-            jobConf.set(BatchConstants.CFG_FACT_DICT_COLUMN_NAMES, joiner.join(factDictColNames));
-            jobConf.set(BatchConstants.CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES, joiner.join(factDictColRowKeyIndexes));
+            attachKylinPropsAndMetadata(cubeInstance, job.getConfiguration());
 
             return waitForCompletion(job);
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
index b2ec6c6..518c7c4 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
@@ -19,6 +19,8 @@
 package org.apache.kylin.job.hadoop.cube;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.ShortWritable;
@@ -32,18 +34,26 @@ import org.apache.kylin.common.mr.KylinMapper;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.RowKeyDesc;
+import org.apache.kylin.dict.DictionaryManager;
 import org.apache.kylin.job.constant.BatchConstants;
 import org.apache.kylin.job.hadoop.AbstractHadoopJob;
 import org.apache.kylin.job.hadoop.hive.CubeJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
 
 /**
  * @author yangli9
  */
 public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRecord, ShortWritable, Text> {
 
+    private String cubeName;
+    private CubeInstance cube;
+    private CubeDesc cubeDesc;
+    private int[] factDictCols;
+
     private CubeJoinedFlatTableDesc intermediateTableDesc;
-    private int[] factDictColRowKeyIndexes;
 
     private ShortWritable outputKey = new ShortWritable();
     private Text outputValue = new Text();
@@ -58,16 +68,31 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
         Configuration conf = context.getConfiguration();
 
         KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
-        String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
-        CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
-        CubeDesc cubeDesc = cube.getDescriptor();
+        cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
+        cube = CubeManager.getInstance(config).getCube(cubeName);
+        cubeDesc = cube.getDescriptor();
         intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
 
-        String[] rowKeyIndexes = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES).split(",");
-        factDictColRowKeyIndexes = new int[rowKeyIndexes.length];
-        for (int i = 0; i < rowKeyIndexes.length; i++) {
-            factDictColRowKeyIndexes[i] = Integer.parseInt(rowKeyIndexes[i]);
+        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+        Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+        List<TblColRef> columns = baseCuboid.getColumns();
+
+        ArrayList<Integer> factDictCols = new ArrayList<Integer>();
+        RowKeyDesc rowkey = cubeDesc.getRowkey();
+        DictionaryManager dictMgr = DictionaryManager.getInstance(config);
+        for (int i = 0; i < columns.size(); i++) {
+            TblColRef col = columns.get(i);
+            if (rowkey.isUseDictionary(col) == false)
+                continue;
+
+            String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
+            if (cubeDesc.getModel().isFactTable(scanTable)) {
+                factDictCols.add(i);
+            }
         }
+        this.factDictCols = new int[factDictCols.size()];
+        for (int i = 0; i < factDictCols.size(); i++)
+            this.factDictCols[i] = factDictCols.get(i);
 
         schema = HCatInputFormat.getTableSchema(context.getConfiguration());
     }
@@ -79,7 +104,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
 
             int[] flatTableIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
             HCatFieldSchema fieldSchema = null;
-            for (int i : factDictColRowKeyIndexes) {
+            for (int i : factDictCols) {
                 outputKey.set((short) i);
                 fieldSchema = schema.get(flatTableIndexes[i]);
                 Object fieldValue = record.get(fieldSchema.getName(), schema);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2bbc467/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index 411f6d9..b161bfa 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -18,13 +18,6 @@
 
 package org.apache.kylin.job.hadoop.cube;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -43,19 +36,18 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.job.constant.BatchConstants;
 import org.apache.kylin.job.hadoop.AbstractHadoopJob;
 import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
 
 /**
  * @author yangli9
  */
 public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text, NullWritable, Text> {
-    private static Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
 
-    private String outputPath;
-    private FileSystem fs;
     private List<TblColRef> columnList = new ArrayList<TblColRef>();
-    private Set<String> toBeExtractedColumns = new HashSet<String>();
 
     @Override
     protected void setup(Context context) throws IOException {
@@ -70,18 +62,11 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
         long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
         Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
         columnList = baseCuboid.getColumns();
-
-        outputPath = conf.get(BatchConstants.OUTPUT_PATH);
-        fs = FileSystem.get(conf);
-
-        String[] factDictColNames = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_NAMES).split(",");
-        toBeExtractedColumns.addAll(Arrays.asList(factDictColNames));
     }
 
     @Override
     public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
         TblColRef col = columnList.get(key.get());
-        toBeExtractedColumns.remove(col.getName());
 
         HashSet<ByteArray> set = new HashSet<ByteArray>();
         for (Text textValue : values) {
@@ -89,6 +74,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
             set.add(value);
         }
 
+        Configuration conf = context.getConfiguration();
+        FileSystem fs = FileSystem.get(conf);
+        String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
         FSDataOutputStream out = fs.create(new Path(outputPath, col.getName()));
 
         try {
@@ -102,15 +90,4 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
 
     }
 
-    @Override
-    protected void cleanup(Context context) throws IOException, InterruptedException {
-        // KYLIN-921 create empty file for dimension with all nulls
-        for (String colName : toBeExtractedColumns) {
-            logger.info("create empty file for column " + colName);
-            boolean success = fs.createNewFile(new Path(outputPath, colName));
-            if (!success) {
-                throw new IOException("failed to create empty file for column " + colName);
-            }
-        }
-    }
 }