You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/09 13:08:13 UTC
incubator-kylin git commit: bug fix on long running cube steaming
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 02b0d84a0 -> 247d4f13c
bug fix on long running cube steaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/247d4f13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/247d4f13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/247d4f13
Branch: refs/heads/0.8.0
Commit: 247d4f13c5bc0d0f910e041bdd34f11e1483fe08
Parents: 02b0d84
Author: honma <ho...@ebay.com>
Authored: Tue Jun 9 19:07:51 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jun 9 19:08:03 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 2 +-
.../java/org/apache/kylin/cube/CubeManager.java | 20 ++++++++++++++++----
.../apache/kylin/query/routing/QueryRouter.java | 12 ++++++------
.../apache/kylin/rest/service/CacheService.java | 1 +
.../apache/kylin/rest/service/CubeService.java | 2 ++
.../kylin/storage/hbase/CubeStorageEngine.java | 9 +++++++--
6 files changed, 33 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 73f708d..01a22c1 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -77,7 +77,7 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
- System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433496233000L));
+ System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433833611000L));
System.out.println(org.apache.kylin.common.util.DateFormat.formatToTimeStr(1433250517000L));
System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-06-01 00:00:00"));
System.out.println(org.apache.kylin.common.util.DateFormat.stringToMillis("2015-05-15 17:00:00"));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index ac49dfa..29bc9b0 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -18,6 +18,8 @@
package org.apache.kylin.cube;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
@@ -48,6 +50,7 @@ import org.apache.kylin.metadata.realization.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -351,7 +354,7 @@ public class CubeManager implements IRealizationProvider {
return appendSegments(cube, startDate, endDate, strictChecking, saveChange);
}
- public CubeSegment appendSegments(CubeInstance cube, long startDate, long endDate, boolean strictChecking, boolean saveChange) throws IOException {
+ public CubeSegment appendSegments(CubeInstance cube, long startDate, long endDate, boolean strictChecking, boolean saveChange) throws IOException {
if (strictChecking)
checkNoBuildingSegment(cube);
@@ -385,7 +388,7 @@ public class CubeManager implements IRealizationProvider {
Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate);
CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());
- validateNewSegments(cube, newSegment);
+ validateNewSegments(cube, false, newSegment);
CubeBuilder cubeBuilder = new CubeBuilder(cube);
cubeBuilder.setToAddSegs(newSegment);
@@ -553,7 +556,7 @@ public class CubeManager implements IRealizationProvider {
for (CubeSegment segment : readySegments) {
long thisSegmentRange = segment.getDateRangeEnd() - segment.getDateRangeStart();
- if (thisSegmentRange >= toMergeRange ) {
+ if (thisSegmentRange >= toMergeRange) {
// this segment and its previous segments will not be merged
toMergeSegments.clear();
currentRange = 0;
@@ -658,7 +661,7 @@ public class CubeManager implements IRealizationProvider {
}
firstSeg.validate();
- for (int i = 0, j = 1; j < tobe.size();) {
+ for (int i = 0, j = 1; j < tobe.size(); ) {
CubeSegment is = tobe.get(i);
CubeSegment js = tobe.get(j);
js.validate();
@@ -750,6 +753,15 @@ public class CubeManager implements IRealizationProvider {
usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier());
}
+ logger.info("Reloaded new cube: " + cubeName + " with reference being" + cubeInstance + " having " + cubeInstance.getSegments().size() + " segments:" +
+ StringUtils.join(Collections2.transform(cubeInstance.getSegments(), new Function<CubeSegment, String>() {
+ @Nullable
+ @Override
+ public String apply(CubeSegment input) {
+ return input.getStorageLocationIdentifier();
+ }
+ }), ","));
+
return cubeInstance;
} catch (Exception e) {
logger.error("Error during load cube instance " + path, e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index e9e7e7b..14db363 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -18,16 +18,15 @@
package org.apache.kylin.query.routing;
-import java.util.List;
-
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
/**
* @author xjiang
@@ -39,6 +38,7 @@ public class QueryRouter {
public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException {
ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig());
+ logger.info("The project manager's reference is " + prjMgr);
String factTableName = olapContext.firstTableScan.getTableName();
String projectName = olapContext.olapSchema.getProjectName();
List<IRealization> realizations = Lists.newArrayList(prjMgr.getRealizationsByTable(projectName, factTableName));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index e580f3a..ea6d410 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -67,6 +67,7 @@ public class CacheService extends BasicService {
public void rebuildCache(Broadcaster.TYPE cacheType, String cacheKey) {
final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey;
+ logger.info(log);
try {
switch (cacheType) {
case CUBE:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 2469ebd..5ea3004 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -242,6 +242,7 @@ public class CubeService extends BasicService {
try {
if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
+ logger.info("Releasing all segments due to cube desc conflict");
this.releaseAllSegments(cube);
}
@@ -385,6 +386,7 @@ public class CubeService extends BasicService {
throw new JobException("Enable is not allowed with a running job.");
}
if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
+ logger.info("Releasing all segments due to cube desc conflict");
cube = this.releaseAllSegments(cube);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/247d4f13/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
index 3689877..4d143d5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java
@@ -144,7 +144,6 @@ public class CubeStorageEngine implements ICachableStorageEngine {
return this.uuid;
}
-
@Override
public boolean isDynamic() {
return false;
@@ -399,8 +398,12 @@ public class CubeStorageEngine implements ICachableStorageEngine {
List<HBaseKeyRange> result = Lists.newArrayList();
+ logger.info("Current cubeInstance is " + cubeInstance + " with " + cubeInstance.getSegments().size() + " segs in all");
+ List<CubeSegment> segs = cubeInstance.getSegments(SegmentStatusEnum.READY);
+ logger.info("READY segs count: " + segs.size());
+
// build row key range for each cube segment
- for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
+ for (CubeSegment cubeSeg : segs) {
// consider derived (lookup snapshot), filter on dimension may
// differ per segment
@@ -420,7 +423,9 @@ public class CubeStorageEngine implements ICachableStorageEngine {
result.addAll(mergedRanges);
}
+ logger.info("hbasekeyrange count: " + result.size());
dropUnhitSegments(result);
+ logger.info("hbasekeyrange count after dropping unhit :" + result.size());
return result;
}