You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/07/03 10:43:59 UTC
incubator-kylin git commit: KYLIN-867 refactor Hybrid
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-867 c87aa4a3b -> 238fb4660
KYLIN-867 refactor Hybrid
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/238fb466
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/238fb466
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/238fb466
Branch: refs/heads/KYLIN-867
Commit: 238fb4660c57eb4c01c27823f0547bc8658ab818
Parents: c87aa4a
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jul 3 16:43:42 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jul 3 16:43:42 2015 +0800
----------------------------------------------------------------------
.../common/util/AbstractKylinTestCase.java | 2 +-
.../org/apache/kylin/cube/CubeInstance.java | 5 -
.../hybrid/test_kylin_hybrid_inner_join.json | 15 +-
.../hybrid/test_kylin_hybrid_left_join.json | 15 +-
.../localmeta/project/default.json | 10 ++
.../apache/kylin/invertedindex/IIInstance.java | 5 -
.../metadata/project/RealizationEntry.java | 22 ++-
.../metadata/realization/IRealization.java | 2 -
.../AdjustForWeeklyMatchedRealization.java | 4 +-
.../RoutingRules/RealizationPriorityRule.java | 4 +-
.../apache/kylin/query/test/IIQueryTest.java | 12 +-
.../apache/kylin/query/test/KylinQueryTest.java | 6 +
.../apache/kylin/rest/service/BasicService.java | 5 +
.../apache/kylin/rest/service/CacheService.java | 4 +
.../apache/kylin/rest/service/CubeService.java | 6 +-
.../kylin/storage/hybrid/HybridInstance.java | 161 +++++++++++--------
.../kylin/storage/hybrid/HybridManager.java | 25 ++-
.../storage/hybrid/HybridStorageEngine.java | 62 +++----
.../kylin/storage/hybrid/HybridManagerTest.java | 9 +-
19 files changed, 207 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index a4a6dc4..d337501 100644
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -36,7 +36,7 @@ public abstract class AbstractKylinTestCase {
public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
- public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.metadata.project.ProjectManager" };
+ public static final String[] SERVICES_WITH_CACHE = { "org.apache.kylin.metadata.MetadataManager", "org.apache.kylin.cube.CubeManager", "org.apache.kylin.cube.CubeDescManager", "org.apache.kylin.invertedindex.IIDescManager", "org.apache.kylin.invertedindex.IIManager", "org.apache.kylin.metadata.realization.RealizationRegistry", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.metadata.project.ProjectManager" };
public abstract void createTestMetadata() throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 898925a..88f110f 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -377,11 +377,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
}
@Override
- public String getModelName() {
- return this.getDescriptor().getModelName();
- }
-
- @Override
public List<TblColRef> getAllDimensions() {
return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
index 5582841..e38e55b 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
@@ -1,14 +1,13 @@
{
"uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
"name": "test_kylin_hybrid_inner_join",
- "historyRealization": {
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- },
- "realTimeRealization": {
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii"
- },
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_empty"
+ }
+ ],
+ "cost": 40,
"last_modified": 1420016227424,
"create_time": null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
index 37f59d6..48cebe6 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
@@ -1,14 +1,13 @@
{
"uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
"name": "test_kylin_hybrid_left_join",
- "historyRealization": {
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_left_join_empty"
- },
- "realTimeRealization": {
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii"
- },
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_left_join_empty"
+ }
+ ],
+ "cost": 40,
"last_modified": 1420016227424,
"create_time": null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index 853a10a..639f3be 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -26,6 +26,16 @@
"name": "test_kylin_ii",
"type": "INVERTED_INDEX",
"realization": "test_kylin_ii"
+ },
+ {
+ "name": "test_kylin_hybrid_inner_join",
+ "type": "HYBRID",
+ "realization": "test_kylin_hybrid_inner_join"
+ },
+ {
+ "name": "test_kylin_hybrid_left_join",
+ "type": "HYBRID",
+ "realization": "test_kylin_hybrid_left_join"
}
]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
index 9c46fbb..89ed059 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
@@ -338,11 +338,6 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
}
@Override
- public String getModelName() {
- return this.getDescriptor().getModelName();
- }
-
- @Override
public List<TblColRef> getAllDimensions() {
return getDescriptor().listAllDimensions();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
index cfbe6a3..73c899b 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
@@ -49,7 +49,27 @@ public class RealizationEntry {
public void setRealization(String realization) {
this.realization = realization;
}
-
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ RealizationEntry entry = (RealizationEntry) o;
+
+ if (realization != null ? !realization.equalsIgnoreCase(entry.realization) : entry.realization != null) return false;
+ if (type != entry.type) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = type != null ? type.hashCode() : 0;
+ result = 31 * result + (realization != null ? realization.hashCode() : 0);
+ return result;
+ }
+
@Override
public String toString() {
return "" + type.name() + "." + realization;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 43f5e9a..1f3837d 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -69,6 +69,4 @@ public interface IRealization {
public long getDateRangeEnd();
- public String getModelName();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
index 3402556..8872a7d 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
@@ -49,8 +49,8 @@ public class AdjustForWeeklyMatchedRealization extends RoutingRule {
if (first instanceof HybridInstance) {
HybridInstance hybrid = (HybridInstance) first;
- if (hybrid.getHistoryRealizationInstance() instanceof CubeInstance)
- first = hybrid.getHistoryRealizationInstance();
+ if (hybrid.getRealizations()[0] instanceof CubeInstance)
+ first = hybrid.getRealizations()[0];
}
if (first instanceof CubeInstance) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
index 7425d62..72995d6 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/RealizationPriorityRule.java
@@ -33,9 +33,9 @@ public class RealizationPriorityRule extends RoutingRule {
static Map<RealizationType, Integer> priorities = Maps.newHashMap();
static {
- priorities.put(RealizationType.HYBRID, 2);
priorities.put(RealizationType.CUBE, 1);
- priorities.put(RealizationType.INVERTED_INDEX, 0);
+ priorities.put(RealizationType.HYBRID, 1);
+ priorities.put(RealizationType.INVERTED_INDEX, 2);
}
public static void setPriorities(Map<RealizationType, Integer> priorities) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
index b58e439..864e3c1 100644
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
@@ -39,9 +39,9 @@ public class IIQueryTest extends KylinQueryTest {
KylinQueryTest.setUp();//invoke super class
Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 0);
- priorities.put(RealizationType.CUBE, 1);
- priorities.put(RealizationType.HYBRID, 1);
+ priorities.put(RealizationType.INVERTED_INDEX, 1);
+ priorities.put(RealizationType.CUBE, 2);
+ priorities.put(RealizationType.HYBRID, 2);
RealizationPriorityRule.setPriorities(priorities);
}
@@ -51,9 +51,9 @@ public class IIQueryTest extends KylinQueryTest {
KylinQueryTest.tearDown();//invoke super class
Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 1);
- priorities.put(RealizationType.CUBE, 0);
- priorities.put(RealizationType.HYBRID, 0);
+ priorities.put(RealizationType.INVERTED_INDEX, 2);
+ priorities.put(RealizationType.CUBE, 1);
+ priorities.put(RealizationType.HYBRID, 1);
RealizationPriorityRule.setPriorities(priorities);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 8c3901c..415783a 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -28,6 +28,7 @@ import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.query.enumerator.OLAPQuery;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
import org.junit.AfterClass;
@@ -105,6 +106,7 @@ public class KylinQueryTest extends KylinTestBase {
protected static void preferCubeOf(String joinType) {
CubeManager cubeManager = CubeManager.getInstance(config);
+ HybridManager hybridManager = HybridManager.getInstance(config);
boolean cubesBuiltInBatch = cubeManager.getCube("test_kylin_cube_with_slr_empty") != null && cubeManager.getCube("test_kylin_cube_without_slr_empty") != null && cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty") != null && cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty") != null;
@@ -114,11 +116,15 @@ public class KylinQueryTest extends KylinTestBase {
}
if (joinType.equals("inner")) {
+ hybridManager.getHybridInstance("test_kylin_hybrid_inner_join").setCost(10);
+ hybridManager.getHybridInstance("test_kylin_hybrid_left_join").setCost(100);
cubeManager.getCube("test_kylin_cube_with_slr_empty").setCost(20);
cubeManager.getCube("test_kylin_cube_without_slr_empty").setCost(10);
cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty").setCost(100);
cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty").setCost(90);
} else if (joinType.equals("left") || joinType.equals("default")) {
+ hybridManager.getHybridInstance("test_kylin_hybrid_left_join").setCost(20);
+ hybridManager.getHybridInstance("test_kylin_hybrid_inner_join").setCost(100);
cubeManager.getCube("test_kylin_cube_with_slr_empty").setCost(100);
cubeManager.getCube("test_kylin_cube_without_slr_empty").setCost(90);
cubeManager.getCube("test_kylin_cube_with_slr_left_join_empty").setCost(20);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index f248e95..2dfc3ec 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -44,6 +44,7 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.rest.controller.QueryController;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict;
@@ -173,6 +174,10 @@ public abstract class BasicService {
return IIManager.getInstance(getConfig());
}
+ public final HybridManager getHybridManager() {
+ return HybridManager.getInstance(getConfig());
+ }
+
protected List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) {
List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllExecutables()).filter(new Predicate<AbstractExecutable>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 9ec1621..f68fd91 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -28,6 +28,7 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.springframework.stereotype.Component;
import java.io.IOException;
@@ -45,6 +46,7 @@ public class CacheService extends BasicService {
switch (cacheType) {
case CUBE:
getCubeManager().loadCubeCache(cacheKey);
+ getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cacheKey);
cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
break;
case CUBE_DESC:
@@ -55,6 +57,7 @@ public class CacheService extends BasicService {
break;
case INVERTED_INDEX:
getIIManager().loadIICache(cacheKey);
+ getHybridManager().reloadHybridInstanceByChild(RealizationType.INVERTED_INDEX, cacheKey);
cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
break;
case INVERTED_INDEX_DESC:
@@ -76,6 +79,7 @@ public class CacheService extends BasicService {
CubeManager.clearCache();
IIDescManager.clearCache();
IIManager.clearCache();
+ HybridManager.clearCache();
RealizationRegistry.clearCache();
ProjectManager.clearCache();
BasicService.resetOLAPDataSources();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index de97a7b..834eb31 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -204,9 +204,9 @@ public class CubeService extends BasicService {
if (project == null) {
return false;
}
- for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
- if (projectDataModel.getType() == RealizationType.CUBE) {
- CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
+ for (RealizationEntry projectRealization : project.getRealizationEntries()) {
+ if (projectRealization != null && projectRealization.getType() == RealizationType.CUBE) {
+ CubeInstance cube = getCubeManager().getCube(projectRealization.getRealization());
assert cube != null;
if (cube.equals(target)) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 1694719..68984cd 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -3,11 +3,10 @@ package org.apache.kylin.storage.hybrid;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -16,6 +15,7 @@ import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.realization.SQLDigest;
+import java.util.LinkedHashSet;
import java.util.List;
/**
@@ -30,40 +30,90 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
@JsonProperty("name")
private String name;
- @JsonProperty("historyRealization")
- private RealizationEntry historyRealization;
+ @JsonProperty("realizations")
+ private List<RealizationEntry> realizationEntries;
- @JsonProperty("realTimeRealization")
- private RealizationEntry realTimeRealization;
+ @JsonProperty("cost")
+ private int cost = 50;
- private IRealization historyRealizationInstance;
- private IRealization realTimeRealizationInstance;
+ private IRealization[] realizations = null;
+ private List<TblColRef> allDimensions = null;
+ private List<TblColRef> allColumns = null;
+ private List<MeasureDesc> allMeasures = null;
+ private long dateRangeStart;
+ private long dateRangeEnd;
+ private boolean isReady = false;
private String projectName;
- public void init() {
- RealizationRegistry registry = RealizationRegistry.getInstance(config);
- historyRealizationInstance = registry.getRealization(historyRealization.getType(), historyRealization.getRealization());
- realTimeRealizationInstance = registry.getRealization(realTimeRealization.getType(), realTimeRealization.getRealization());
+ private boolean initiated = false;
- if (historyRealizationInstance == null) {
- throw new IllegalArgumentException("Didn't find realization '" + historyRealization.getType() + "'" + " with name '" + historyRealization.getRealization() + "' in '" + name + "'");
- }
+ public List<RealizationEntry> getRealizationEntries() {
+ return realizationEntries;
+ }
- if (realTimeRealizationInstance == null) {
- throw new IllegalArgumentException("Didn't find realization '" + realTimeRealization.getType() + "'" + " with name '" + realTimeRealization.getRealization() + "' in '" + name + "'");
- }
+ private void init() {
+ if (initiated == true)
+ return;
+
+ synchronized (this) {
+ if (initiated == true)
+ return;
+
+ if (realizationEntries == null || realizationEntries.size() == 0)
+ throw new IllegalArgumentException();
+
+ RealizationRegistry registry = RealizationRegistry.getInstance(config);
+ realizations = new IRealization[realizationEntries.size()];
+ for (int i = 0; i < realizationEntries.size(); i++) {
+ IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), realizationEntries.get(i).getRealization());
+ if (realization == null)
+ throw new IllegalArgumentException("Realization '" + realizationEntries.get(i) + "' not loaded.");
+ realizations[i] = realization;
+ }
+
+ LinkedHashSet<TblColRef> columns = new LinkedHashSet<TblColRef>();
+ LinkedHashSet<TblColRef> dimensions = new LinkedHashSet<TblColRef>();
+ LinkedHashSet<MeasureDesc> measures = new LinkedHashSet<MeasureDesc>();
+ dateRangeStart = 0;
+ dateRangeEnd = Long.MAX_VALUE;
+ for (IRealization realization : realizations) {
+ if (realization.isReady() == false)
+ continue;
+
+ columns.addAll(realization.getAllColumns());
+ dimensions.addAll(realization.getAllDimensions());
+ measures.addAll(realization.getMeasures());
+
+ if (realization.isReady())
+ isReady = true;
+ if (dateRangeStart == 0 || realization.getDateRangeStart() < dateRangeStart)
+ dateRangeStart = realization.getDateRangeStart();
+
+ if (dateRangeStart == Long.MAX_VALUE || realization.getDateRangeEnd() > dateRangeEnd)
+ dateRangeEnd = realization.getDateRangeEnd();
+ }
+
+ allDimensions = Lists.newArrayList(dimensions);
+ allColumns = Lists.newArrayList(columns);
+ allMeasures = Lists.newArrayList(measures);
+
+ initiated = true;
+ }
}
@Override
public boolean isCapable(SQLDigest digest) {
- return getHistoryRealizationInstance().isCapable(digest) && getRealTimeRealizationInstance().isCapable(digest);
+ for (IRealization realization : getRealizations()) {
+ if (realization.isCapable(digest))
+ return true;
+ }
+ return false;
}
@Override
public int getCost(SQLDigest digest) {
- return historyRealizationInstance.getCost(digest);
- //return Math.min(historyRealizationInstance.getCost(digest), realTimeRealizationInstance.getCost(digest));
+ return cost;
}
@Override
@@ -73,22 +123,24 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
@Override
public String getFactTable() {
- return getHistoryRealizationInstance().getFactTable();
+ return getRealizations()[0].getFactTable();
}
@Override
public List<TblColRef> getAllColumns() {
- return getHistoryRealizationInstance().getAllColumns();
+ init();
+ return allColumns;
}
@Override
public List<MeasureDesc> getMeasures() {
- return getHistoryRealizationInstance().getMeasures();
+ init();
+ return allMeasures;
}
@Override
public boolean isReady() {
- return historyRealizationInstance.isReady() || realTimeRealizationInstance.isReady();
+ return isReady;
}
@Override
@@ -128,61 +180,32 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
this.config = config;
}
- public RealizationEntry getHistoryRealization() {
- return historyRealization;
- }
-
- public RealizationEntry getRealTimeRealization() {
- return realTimeRealization;
- }
-
- public IRealization getHistoryRealizationInstance() {
- if (historyRealizationInstance == null) {
- this.init();
- }
- return historyRealizationInstance;
- }
-
- public IRealization getRealTimeRealizationInstance() {
- if (realTimeRealizationInstance == null) {
- this.init();
- }
- return realTimeRealizationInstance;
- }
-
@Override
public long getDateRangeStart() {
- return Math.min(getHistoryRealizationInstance().getDateRangeStart(), getRealTimeRealizationInstance().getDateRangeStart());
+ return dateRangeStart;
}
@Override
public long getDateRangeEnd() {
- return Math.max(getHistoryRealizationInstance().getDateRangeEnd(), getRealTimeRealizationInstance().getDateRangeEnd()) +1;
+ return dateRangeEnd;
}
-
-
- public DataModelDesc getDataModelDesc(){
- if (getHistoryRealizationInstance() instanceof CubeInstance) {
- return ((CubeInstance) historyRealizationInstance).getDescriptor().getModel();
- }
-
- return ((IIInstance) historyRealizationInstance).getDescriptor().getModel();
+ @Override
+ public List<TblColRef> getAllDimensions() {
+ init();
+ return allDimensions;
}
- @Override
- public String getModelName() {
- if (getHistoryRealizationInstance() instanceof CubeInstance) {
- return ((CubeInstance) historyRealizationInstance).getDescriptor().getModelName();
- }
-
- return ((IIInstance) historyRealizationInstance).getDescriptor().getModelName();
+ public IRealization[] getRealizations() {
+ init();
+ return realizations;
}
- @Override
- public List<TblColRef> getAllDimensions(){
-
- return this.getHistoryRealizationInstance().getAllDimensions();
+ public static String concatResourcePath(String hybridName) {
+ return ResourceStore.HYBRID_RESOURCE_ROOT + "/" + hybridName + ".json";
}
+ public void setCost(int cost) {
+ this.cost = cost;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index df3fae3..8373cf1 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -81,6 +81,21 @@ public class HybridManager implements IRealizationProvider {
logger.debug("Loaded " + paths.size() + " Hybrid(s)");
}
+ public void reloadHybridInstanceByChild(RealizationType type, String realizationName) throws IOException {
+ for (HybridInstance hybridInstance : hybridMap.values()) {
+ boolean includes = false;
+ for (IRealization realization : hybridInstance.getRealizations()) {
+ if (realization.getType() == type && realization.getName().equalsIgnoreCase(realizationName)) {
+ includes = true;
+ break;
+ }
+ }
+
+ if (includes == true)
+ loadHybridInstance(HybridInstance.concatResourcePath(hybridInstance.getName()));
+ }
+ }
+
private synchronized HybridInstance loadHybridInstance(String path) throws IOException {
ResourceStore store = getStore();
@@ -89,14 +104,13 @@ public class HybridManager implements IRealizationProvider {
hybridInstance = store.getResource(path, HybridInstance.class, HYBRID_SERIALIZER);
hybridInstance.setConfig(config);
+ if (hybridInstance.getRealizationEntries() == null || hybridInstance.getRealizationEntries().size() == 0) {
+ throw new IllegalStateException("HybridInstance must have realization entries, " + path);
+ }
+
if (StringUtils.isBlank(hybridInstance.getName()))
throw new IllegalStateException("HybridInstance name must not be blank, at " + path);
- if (hybridInstance.getHistoryRealization() == null || hybridInstance.getRealTimeRealization() == null) {
-
- throw new IllegalStateException("HybridInstance must have both historyRealization and realTimeRealization set, at " + path);
- }
-
final String name = hybridInstance.getName();
hybridMap.putLocal(name, hybridInstance);
@@ -124,4 +138,5 @@ public class HybridManager implements IRealizationProvider {
private ResourceStore getStore() {
return ResourceStore.getStore(this.config);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 7ad5ed0..11ad211 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,69 +1,43 @@
package org.apache.kylin.storage.hybrid;
-import com.google.common.base.Function;
import com.google.common.collect.Lists;
-import com.google.common.collect.Ranges;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.realization.SQLDigestUtil;
import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.IStorageEngine;
import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageEngineFactory;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import javax.annotation.Nullable;
+import java.util.List;
/**
*/
public class HybridStorageEngine implements IStorageEngine {
- private HybridInstance hybridInstance;
- private IStorageEngine historicalStorageEngine;
- private IStorageEngine realtimeStorageEngine;
+ private IRealization[] realizations;
+ private IStorageEngine[] storageEngines;
public HybridStorageEngine(HybridInstance hybridInstance) {
- this.hybridInstance = hybridInstance;
- this.historicalStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getHistoryRealizationInstance());
- this.realtimeStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getRealTimeRealizationInstance());
+ this.realizations = hybridInstance.getRealizations();
+ storageEngines = new IStorageEngine[realizations.length];
+ for (int i = 0; i < realizations.length; i++) {
+ storageEngines[i] = StorageEngineFactory.getStorageEngine(realizations[i]);
+ }
}
@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest) {
-
- // search the historic realization
- ITupleIterator historicalDataIterator = this.historicalStorageEngine.search(context, sqlDigest);
-
- String modelName = hybridInstance.getModelName();
- MetadataManager metaMgr = getMetadataManager();
- DataModelDesc modelDesc = metaMgr.getDataModelDesc(modelName);
-
- // if the model isn't partitioned, only query the history
- if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null)
- return historicalDataIterator;
-
- TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
-
- ITupleIterator realtimeDataIterator = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, //
- Ranges.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),//
- new Function<Void, ITupleIterator>() {
- @Nullable
- @Override
- public ITupleIterator apply(@Nullable Void input) {
- return realtimeStorageEngine.search(context, sqlDigest);
- }
- });
-
- // combine history and real-time tuple iterator
- return new CompoundTupleIterator(Lists.newArrayList(historicalDataIterator, realtimeDataIterator));
+ List<ITupleIterator> tupleIterators = Lists.newArrayList();
+ for (int i = 0; i < realizations.length; i++) {
+ if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest)) {
+ ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest);
+ tupleIterators.add(dataIterator);
+ }
+ }
+ // combine tuple iterator
+ return new CompoundTupleIterator(tupleIterators);
}
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/238fb466/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
index 3ad7ca6..8a1bcf0 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -26,15 +26,12 @@ public class HybridManagerTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() throws Exception {
- HybridInstance cube = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join");
- cube.init();
- System.out.println(JsonUtil.writeValueAsIndentString(cube));
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join");
+ System.out.println(JsonUtil.writeValueAsIndentString(hybridInstance));
- IRealization history = cube.getHistoryRealizationInstance();
- IRealization realTime = cube.getRealTimeRealizationInstance();
+ IRealization history = hybridInstance.getRealizations()[0];
Assert.assertTrue(history instanceof CubeInstance);
- Assert.assertTrue(realTime instanceof IIInstance);
}