You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/29 08:43:39 UTC
incubator-kylin git commit: concrete views
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-942 80208e052 -> d927ea124
concrete views
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d927ea12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d927ea12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d927ea12
Branch: refs/heads/KYLIN-942
Commit: d927ea124091bb14abe9f560bc74611f14771ba3
Parents: 80208e0
Author: honma <ho...@ebay.com>
Authored: Tue Sep 29 14:46:47 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 29 14:46:47 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/common/CuboidShardUtil.java | 4 ++--
.../apache/kylin/storage/hbase/steps/CreateHTableJob.java | 9 ++++-----
2 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d927ea12/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
index 507f5c4..e1af752 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidShardUtil.java
@@ -40,7 +40,7 @@ public class CuboidShardUtil {
public static void saveCuboidShards(CubeSegment segment, Map<Long, Short> cuboidShards, int totalShards) throws IOException {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<Long, Short> filered = Maps.filterEntries(cuboidShards, new Predicate<Map.Entry<Long, Short>>() {
+ Map<Long, Short> filered = Maps.newHashMap(Maps.filterEntries(cuboidShards, new Predicate<Map.Entry<Long, Short>>() {
@Override
public boolean apply(Map.Entry<Long, Short> input) {
if (input.getValue() <= 1) {
@@ -51,7 +51,7 @@ public class CuboidShardUtil {
return true;
}
}
- });
+ }));
segment.setCuboidShardNums(filered);
segment.setTotalShards(totalShards);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d927ea12/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index cc46483..b3b6843 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -256,12 +256,12 @@ public class CreateHTableJob extends AbstractHadoopJob {
allCuboids.addAll(cubeRowCountMap.keySet());
Collections.sort(allCuboids);
- Map<Long, Double> cubeSizeMap = Maps.transformEntries(cubeRowCountMap, new Maps.EntryTransformer<Long, Long, Double>() {
+ Map<Long, Double> cubeSizeMap = Maps.newHashMap(Maps.transformEntries(cubeRowCountMap, new Maps.EntryTransformer<Long, Long, Double>() {
@Override
public Double transformEntry(@Nullable Long key, @Nullable Long value) {
return estimateCuboidStorageSize(cubeDesc, key, value, baseCuboidId, rowkeyColumnSize);
}
- });
+ }));
for (Double cuboidSize : cubeSizeMap.values()) {
totalSizeInM += cuboidSize;
}
@@ -298,7 +298,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
double[] regionSizes = new double[nRegion];
for (long cuboidId : allCuboids) {
double estimatedSize = cubeSizeMap.get(cuboidId);
- double magic = 13;
+ double magic = 23;
int shardNum = (int) (1.0 * estimatedSize * magic / mbPerRegion);
if (shardNum < 1) {
shardNum = 1;
@@ -390,9 +390,8 @@ public class CreateHTableJob extends AbstractHadoopJob {
}
bytesLength += space;
- logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + bytesLength + " bytes.");
double ret = 1.0 * bytesLength * rowCount / (1024L * 1024L);
- logger.info("Cuboid " + cuboidId + " total size is " + ret + "M.");
+ logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + bytesLength + " bytes." + " Total size is " + ret + "M.");
return ret;
}