You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/08/07 09:53:45 UTC
[kylin] branch master updated: KYLIN-3463 use LazyOutputFormat to
prevent to create zero-sized default output for optimize jobs
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new e7bacd3 KYLIN-3463 use LazyOutputFormat to prevent to create zero-sized default output for optimize jobs
e7bacd3 is described below
commit e7bacd35ea771db6ebaa2dad8d3a8994f76a78d3
Author: Zhong <nj...@apache.org>
AuthorDate: Mon Jul 23 14:37:34 2018 +0800
KYLIN-3463 use LazyOutputFormat to prevent to create zero-sized default output for optimize jobs
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java | 4 +++-
.../org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java
index 97f9dc1..b56434b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
@@ -78,7 +79,8 @@ public class FilterRecommendCuboidDataJob extends AbstractHadoopJob {
job.setInputFormatClass(SequenceFileInputFormat.class);
FileInputFormat.setInputPaths(job, input);
// Output
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ //// prevent to create zero-sized default output
+ LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
FileOutputFormat.setOutputPath(job, output);
// set job configuration
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java
index 0cd7264..b71e459 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
@@ -80,7 +81,8 @@ public class UpdateOldCuboidShardJob extends AbstractHadoopJob {
job.setInputFormatClass(SequenceFileInputFormat.class);
FileInputFormat.setInputPaths(job, input);
// Output
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
+ //// prevent to create zero-sized default output
+ LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
FileOutputFormat.setOutputPath(job, output);
// set job configuration