You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/08/03 16:21:09 UTC
incubator-kylin git commit: KYLIN-921 take care of situations where
no dict dim exist on fact table
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-921 734fc1ab8 -> 69d686da3
KYLIN-921 take care of situations where no dict dim exist on fact table
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/69d686da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/69d686da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/69d686da
Branch: refs/heads/KYLIN-921
Commit: 69d686da33aaaacac7776a3b142619aa06bb7339
Parents: 734fc1a
Author: honma <ho...@ebay.com>
Authored: Mon Aug 3 22:21:37 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Aug 3 22:21:37 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cube/FactDistinctColumnsMapper.java | 17 +++++++++++------
.../hadoop/cube/FactDistinctColumnsReducer.java | 14 +++++++++-----
2 files changed, 20 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69d686da/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
index 9345171..ca0671d 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapper.java
@@ -20,17 +20,17 @@ package org.apache.kylin.job.hadoop.cube;
import java.io.IOException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.mr.KylinMapper;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -64,10 +64,15 @@ public class FactDistinctColumnsMapper<KEYIN> extends KylinMapper<KEYIN, HCatRec
CubeDesc cubeDesc = cube.getDescriptor();
intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);
- String[] rowKeyIndexes = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES).split(",");
- factDictColRowKeyIndexes = new int[rowKeyIndexes.length];
- for (int i = 0; i < rowKeyIndexes.length; i++) {
- factDictColRowKeyIndexes[i] = Integer.parseInt(rowKeyIndexes[i]);
+ String rowKeyIndexesStr = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_ROWKEY_INDEXES);
+ if (!StringUtils.isEmpty(rowKeyIndexesStr)) {
+ String[] rowKeyIndexes = rowKeyIndexesStr.split(",");
+ factDictColRowKeyIndexes = new int[rowKeyIndexes.length];
+ for (int i = 0; i < rowKeyIndexes.length; i++) {
+ factDictColRowKeyIndexes[i] = Integer.parseInt(rowKeyIndexes[i]);
+ }
+ } else {
+ factDictColRowKeyIndexes = new int[0];
}
schema = HCatInputFormat.getTableSchema(context.getConfiguration());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69d686da/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
index ba212e6..7e24484 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsReducer.java
@@ -19,21 +19,24 @@
package org.apache.kylin.job.hadoop.cube;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.kylin.common.util.Bytes;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.mr.KylinReducer;
import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -72,7 +75,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<ShortWritable, Text
outputPath = conf.get(BatchConstants.OUTPUT_PATH);
fs = FileSystem.get(conf);
- String[] factDictColNames = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_NAMES).split(",");
+ String factDictColNamesStr = conf.get(BatchConstants.CFG_FACT_DICT_COLUMN_NAMES);
+ String[] factDictColNames = StringUtils.isEmpty(factDictColNamesStr) ? new String[0] : factDictColNamesStr.split(",");
toBeExtractedColumns.addAll(Arrays.asList(factDictColNames));
}