You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/29 08:58:17 UTC
incubator-kylin git commit: remove views
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-942 d927ea124 -> b8b3c02cf
remove views
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b8b3c02c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b8b3c02c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b8b3c02c
Branch: refs/heads/KYLIN-942
Commit: b8b3c02cf5701804005477d5a4d248679f84d845
Parents: d927ea1
Author: honma <ho...@ebay.com>
Authored: Tue Sep 29 15:01:25 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 29 15:01:25 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 6 ++---
.../kylin/engine/mr/common/CuboidShardUtil.java | 26 ++++++++------------
.../storage/hbase/steps/CreateHTableJob.java | 11 ++++-----
3 files changed, 18 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b8b3c02c/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 1590c92..cd83d96 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -25,6 +25,7 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.IdentityHashMap;
+import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
import org.junit.Assert;
@@ -32,7 +33,9 @@ import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.TreeMultiset;
/**
@@ -64,9 +67,6 @@ public class BasicTest {
@Test
public void testxx() {
- short x = -64;
- int y = (int)x;
- System.out.println(y);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b8b3c02c/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
index e1af752..d09e4ec 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
@@ -28,30 +28,24 @@ import org.apache.kylin.cube.CubeUpdate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Predicate;
import com.google.common.collect.Maps;
public class CuboidShardUtil {
protected static final Logger logger = LoggerFactory.getLogger(CuboidShardUtil.class);
- // public static Map<Long, Short> loadCuboidShards(CubeSegment segment) {
- // return DefaultedMap.defaultedMap(segment.getCuboidShards(), (short) 1);
- // }
-
public static void saveCuboidShards(CubeSegment segment, Map<Long, Short> cuboidShards, int totalShards) throws IOException {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<Long, Short> filered = Maps.newHashMap(Maps.filterEntries(cuboidShards, new Predicate<Map.Entry<Long, Short>>() {
- @Override
- public boolean apply(Map.Entry<Long, Short> input) {
- if (input.getValue() <= 1) {
- logger.info("Cuboid {} has {} shards, skip saving it as an optimization", input.getKey(), input.getValue());
- return false;
- } else {
- logger.info("Cuboid {} has {} shards, saving it", input.getKey(), input.getValue());
- return true;
- }
+
+ Map<Long, Short> filered = Maps.newHashMap();
+ for (Map.Entry<Long, Short> entry : cuboidShards.entrySet()) {
+ if (entry.getValue() <= 1) {
+ logger.info("Cuboid {} has {} shards, skip saving it as an optimization", entry.getKey(), entry.getValue());
+ } else {
+ logger.info("Cuboid {} has {} shards, saving it", entry.getKey(), entry.getValue());
+ filered.put(entry.getKey(), entry.getValue());
}
- }));
+ }
+
segment.setCuboidShardNums(filered);
segment.setTotalShards(totalShards);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b8b3c02c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index b3b6843..6a4373e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -256,12 +256,11 @@ public class CreateHTableJob extends AbstractHadoopJob {
allCuboids.addAll(cubeRowCountMap.keySet());
Collections.sort(allCuboids);
- Map<Long, Double> cubeSizeMap = Maps.newHashMap(Maps.transformEntries(cubeRowCountMap, new Maps.EntryTransformer<Long, Long, Double>() {
- @Override
- public Double transformEntry(@Nullable Long key, @Nullable Long value) {
- return estimateCuboidStorageSize(cubeDesc, key, value, baseCuboidId, rowkeyColumnSize);
- }
- }));
+ Map<Long, Double> cubeSizeMap = Maps.newHashMap();
+ for (Map.Entry<Long, Long> entry : cubeRowCountMap.entrySet()) {
+ cubeSizeMap.put(entry.getKey(),estimateCuboidStorageSize(cubeDesc, entry.getKey(), entry.getValue(), baseCuboidId, rowkeyColumnSize));
+ }
+
for (Double cuboidSize : cubeSizeMap.values()) {
totalSizeInM += cuboidSize;
}