You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/28 01:04:52 UTC
[15/50] incubator-kylin git commit: Use LongWritable as key type in
fact distinct job.
Use LongWritable as key type in fact distinct job.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8d40a578
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8d40a578
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8d40a578
Branch: refs/heads/streaming-localdict
Commit: 8d40a578170da66c61503d7b42fe70c3a930dadd
Parents: 7658a50
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu Mar 26 23:32:56 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu Mar 26 23:32:56 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cube/FactDistinctColumnsCombiner.java | 6 +++---
.../kylin/job/hadoop/cube/FactDistinctColumnsJob.java | 4 ++--
.../job/hadoop/cube/FactDistinctColumnsMapperBase.java | 11 +++++------
3 files changed, 10 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d40a578/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
index c0cdd46..59ccd5a 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsCombiner.java
@@ -21,9 +21,9 @@ package org.apache.kylin.job.hadoop.cube;
import java.io.IOException;
import java.util.HashSet;
+import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.common.mr.KylinReducer;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.util.ByteArray;
@@ -31,7 +31,7 @@ import org.apache.kylin.common.util.ByteArray;
/**
* @author yangli9
*/
-public class FactDistinctColumnsCombiner extends KylinReducer<ShortWritable, Text, ShortWritable, Text> {
+public class FactDistinctColumnsCombiner extends KylinReducer<LongWritable, Text, LongWritable, Text> {
private Text outputValue = new Text();
@@ -41,7 +41,7 @@ public class FactDistinctColumnsCombiner extends KylinReducer<ShortWritable, Tex
}
@Override
- public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+ public void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
if(key.get() >= 0) {
HashSet<ByteArray> set = new HashSet<ByteArray>();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d40a578/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index 17c5e9b..5903c7b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import org.apache.commons.cli.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -103,7 +103,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job.setInputFormatClass(HCatInputFormat.class);
job.setMapperClass(FactDistinctHiveColumnsMapper.class);
job.setCombinerClass(FactDistinctColumnsCombiner.class);
- job.setMapOutputKeyClass(ShortWritable.class);
+ job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d40a578/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
index 9945769..2f046ab 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsMapperBase.java
@@ -1,12 +1,7 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.kylin.common.KylinConfig;
@@ -21,6 +16,10 @@ import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Created by Hongbin Ma(Binmahone) on 3/26/15.
*/
@@ -35,7 +34,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
protected LongWritable outputKey = new LongWritable();
protected Text outputValue = new Text();
- protected int errorRecordCounter =0;
+ protected int errorRecordCounter = 0;
@Override
protected void setup(Context context) throws IOException {