You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/24 10:35:46 UTC
incubator-kylin git commit: KYLIN-980 check available memory and then
spill
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging c3880be69 -> 1923cd429
KYLIN-980 check available memory and then spill
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1923cd42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1923cd42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1923cd42
Branch: refs/heads/2.x-staging
Commit: 1923cd429470498b58c67ce39e7160bb658b8a97
Parents: c3880be
Author: shaofengshi <sh...@apache.org>
Authored: Tue Nov 24 17:35:23 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Nov 24 17:35:23 2015 +0800
----------------------------------------------------------------------
.../engine/mr/steps/FactDistinctColumnsReducer.java | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1923cd42/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 66d63fb..ddd366c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -19,26 +19,22 @@
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
@@ -92,14 +88,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
TblColRef col = columnList.get((int) key.get());
HashSet<ByteArray> set = new HashSet<ByteArray>();
-
- Text textValue = null;
- Iterator<Text> valueItr = values.iterator();
- while (valueItr.hasNext()) {
- textValue = valueItr.next();
+ int count = 0;
+ for (Text textValue : values) {
ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
set.add(value);
- if (set.size() >= 5000000) { // output when count reach 5 Million
+ count++;
+ if (count % 10000 == 0 && MemoryBudgetController.getSystemAvailMB() < 100) {
outputDistinctValues(col, set, context);
set.clear();
}