You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/07/08 11:50:41 UTC
[2/2] incubator-kylin git commit: KYLIN-867 Hybrid model for multiple
realizations
KYLIN-867 Hybrid model for multiple realizations
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f8c2283d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f8c2283d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f8c2283d
Branch: refs/heads/0.8
Commit: f8c2283d1886c7d094ed69baa15d5cd8ae1cc06b
Parents: 7895199
Author: shaofengshi <sh...@apache.org>
Authored: Wed Jul 8 14:50:56 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jul 8 17:50:28 2015 +0800
----------------------------------------------------------------------
.../common/util/AbstractKylinTestCase.java | 1 +
.../org/apache/kylin/cube/CubeInstance.java | 1 +
.../hybrid/test_kylin_hybrid_inner_join.json | 18 +-
.../hybrid/test_kylin_hybrid_left_join.json | 18 +-
.../hybrid/test_kylin_hybrid_ready.json | 17 ++
.../apache/kylin/invertedindex/IIInstance.java | 1 +
.../apache/kylin/metadata/MetadataManager.java | 4 +-
.../metadata/project/RealizationEntry.java | 23 ++-
.../metadata/realization/IRealization.java | 2 -
.../query/enumerator/LookupTableEnumerator.java | 13 +-
.../AdjustForWeaklyMatchedRealization.java | 4 +-
.../apache/kylin/rest/service/BasicService.java | 5 +
.../apache/kylin/rest/service/CacheService.java | 5 +
.../kylin/storage/hybrid/HybridInstance.java | 203 +++++++++++++------
.../kylin/storage/hybrid/HybridManager.java | 25 ++-
.../storage/hybrid/HybridStorageEngine.java | 61 ++----
.../kylin/storage/hybrid/HybridManagerTest.java | 15 +-
17 files changed, 266 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index f196744..c9e785a 100644
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -41,6 +41,7 @@ public abstract class AbstractKylinTestCase {
"org.apache.kylin.cube.CubeDescManager", //
"org.apache.kylin.invertedindex.IIDescManager",//
"org.apache.kylin.invertedindex.IIManager",//
+ "org.apache.kylin.storage.hybrid.HybridManager",
"org.apache.kylin.metadata.realization.RealizationRegistry", //
"org.apache.kylin.metadata.project.ProjectManager", //
"org.apache.kylin.metadata.MetadataManager" //
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index f62904e..a56b812 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -120,6 +120,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
return CubeDescManager.getInstance(config).getCubeDesc(descName);
}
+ @Override
public DataModelDesc getDataModelDesc(){return this.getDescriptor().getModel();}
public boolean isReady() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
index e7ae54e..c836ac9 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_inner_join.json
@@ -1,14 +1,16 @@
{
"uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
"name": "test_kylin_hybrid_inner_join",
- "historyRealization": {
- "type": "CUBE",
- "realization": "test_kylin_cube_without_slr_empty"
- },
- "realTimeRealization": {
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii_inner_join"
- },
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_empty"
+ },
+ {
+ "type": "INVERTED_INDEX",
+ "realization": "test_kylin_ii_inner_join"
+ }
+ ],
"last_modified": 1420016227424,
"create_time": null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
index 6390891..95f7d05 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_left_join.json
@@ -1,14 +1,16 @@
{
"uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
"name": "test_kylin_hybrid_left_join",
- "historyRealization": {
- "type": "CUBE",
- "realization": "test_kylin_cube_with_slr_left_join_empty"
- },
- "realTimeRealization": {
- "type": "INVERTED_INDEX",
- "realization": "test_kylin_ii_left_join"
- },
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_left_join_empty"
+ },
+ {
+ "type": "INVERTED_INDEX",
+ "realization": "test_kylin_ii_left_join"
+ }
+ ],
"last_modified": 1420016227424,
"create_time": null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
new file mode 100644
index 0000000..0274a69
--- /dev/null
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid_ready.json
@@ -0,0 +1,17 @@
+{
+ "uuid": "9iiu8590-64b6-4367-8fb5-7500eb95fd9c",
+ "name": "test_kylin_hybrid_ready",
+ "realizations": [
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_ready_2_segments"
+ },
+ {
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_ready"
+ }
+ ],
+ "cost": 100,
+ "last_modified": 1420016227424,
+ "create_time": null
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
index cdb3fa7..faf9079 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
@@ -110,6 +110,7 @@ public class IIInstance extends RootPersistentEntity implements IRealization {
return IIDescManager.getInstance(config).getIIDesc(descName);
}
+ @Override
public DataModelDesc getDataModelDesc(){
return this.getDescriptor().getModel();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 2833218..38b9415 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -310,8 +310,8 @@ public class MetadataManager {
IRealization rel = registry.getRealization(realization.getType(), realization.getRealization());
if (rel != null) {
DataModelDesc modelDesc = rel.getDataModelDesc();
- if(!ret.contains(modelDesc)){
- ProjectManager.getInstance(config).updateModelToProject(modelDesc.getName(),projectName);
+ if (modelDesc != null && !ret.contains(modelDesc)) {
+ ProjectManager.getInstance(config).updateModelToProject(modelDesc.getName(), projectName);
ret.add(modelDesc);
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
index 910d242..7541d77 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
@@ -48,7 +48,28 @@ public class RealizationEntry {
public void setRealization(String realization) {
this.realization = realization;
}
-
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ RealizationEntry entry = (RealizationEntry) o;
+
+ if (realization != null ? !realization.equalsIgnoreCase(entry.realization) : entry.realization != null)
+ return false;
+ if (type != entry.type) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = type != null ? type.hashCode() : 0;
+ result = 31 * result + (realization != null ? realization.hashCode() : 0);
+ return result;
+ }
+
@Override
public String toString() {
return "" + type.name() + "." + realization;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 251e794..ef80b39 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -73,6 +73,4 @@ public interface IRealization {
public long getDateRangeEnd();
public String getModelName();
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index b19061b..2badd30 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -24,6 +24,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPTable;
import org.apache.kylin.storage.hybrid.HybridInstance;
@@ -48,10 +49,16 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
//TODO: assuming LookupTableEnumerator is handled by a cube
CubeInstance cube = null;
- if (olapContext.realization instanceof CubeInstance) {
+ if (olapContext.realization instanceof CubeInstance)
cube = (CubeInstance) olapContext.realization;
- } else if (olapContext.realization instanceof HybridInstance) {
- cube = (CubeInstance) ((HybridInstance) olapContext.realization).getHistoryRealizationInstance();
+ else if (olapContext.realization instanceof HybridInstance) {
+ final HybridInstance hybridInstance = (HybridInstance)olapContext.realization;
+ final IRealization latestRealization = hybridInstance.getLatestRealization();
+ if (latestRealization instanceof CubeInstance) {
+ cube = (CubeInstance) latestRealization;
+ } else {
+ throw new IllegalStateException();
+ }
}
String lookupTableName = olapContext.firstTableScan.getTableName();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
index 0840d06..9fdc35d 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRules/AdjustForWeaklyMatchedRealization.java
@@ -48,9 +48,7 @@ public class AdjustForWeaklyMatchedRealization extends RoutingRule {
if (first instanceof HybridInstance) {
HybridInstance hybrid = (HybridInstance) first;
-
- if (hybrid.getHistoryRealizationInstance() instanceof CubeInstance)
- first = hybrid.getHistoryRealizationInstance();
+ first = hybrid.getLatestRealization();
}
if (first instanceof CubeInstance) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 2eb8472..68b81dd 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -56,6 +56,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -174,6 +175,10 @@ public abstract class BasicService {
return ProjectManager.getInstance(getConfig());
}
+ public final HybridManager getHybridManager() {
+ return HybridManager.getInstance(getConfig());
+ }
+
public final ExecutableManager getExecutableManager() {
return ExecutableManager.getInstance(getConfig());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 3edd3bb..ba3f990 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -31,6 +31,8 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -70,6 +72,7 @@ public class CacheService extends BasicService {
switch (cacheType) {
case CUBE:
CubeInstance newCube = getCubeManager().reloadCubeLocal(cacheKey);
+ getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cacheKey);
getProjectManager().clearL2Cache();
//clean query related cache first
super.cleanDataCache(newCube.getUuid());
@@ -85,6 +88,7 @@ public class CacheService extends BasicService {
case INVERTED_INDEX:
//II update does not need to update storage cache because it is dynamic already
getIIManager().reloadIILocal(cacheKey);
+ getHybridManager().reloadHybridInstanceByChild(RealizationType.INVERTED_INDEX, cacheKey);
getProjectManager().clearL2Cache();
break;
case INVERTED_INDEX_DESC:
@@ -106,6 +110,7 @@ public class CacheService extends BasicService {
CubeManager.clearCache();
IIDescManager.clearCache();
IIManager.clearCache();
+ HybridManager.clearCache();
RealizationRegistry.clearCache();
ProjectManager.clearCache();
super.cleanAllDataCache();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 1694719..d1f5700 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -3,10 +3,10 @@ package org.apache.kylin.storage.hybrid;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -15,8 +15,10 @@ import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.util.List;
+import java.util.*;
/**
*/
@@ -30,40 +32,117 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
@JsonProperty("name")
private String name;
- @JsonProperty("historyRealization")
- private RealizationEntry historyRealization;
+ @JsonProperty("realizations")
+ private List<RealizationEntry> realizationEntries;
- @JsonProperty("realTimeRealization")
- private RealizationEntry realTimeRealization;
+ @JsonProperty("cost")
+ private int cost = 50;
- private IRealization historyRealizationInstance;
- private IRealization realTimeRealizationInstance;
+ private IRealization[] realizations = null;
+ private List<TblColRef> allDimensions = null;
+ private List<TblColRef> allColumns = null;
+ private List<MeasureDesc> allMeasures = null;
+ private long dateRangeStart;
+ private long dateRangeEnd;
+ private boolean isReady = false;
private String projectName;
- public void init() {
- RealizationRegistry registry = RealizationRegistry.getInstance(config);
- historyRealizationInstance = registry.getRealization(historyRealization.getType(), historyRealization.getRealization());
- realTimeRealizationInstance = registry.getRealization(realTimeRealization.getType(), realTimeRealization.getRealization());
+ private boolean initiated = false;
+ private final static Logger logger = LoggerFactory.getLogger(HybridInstance.class);
- if (historyRealizationInstance == null) {
- throw new IllegalArgumentException("Didn't find realization '" + historyRealization.getType() + "'" + " with name '" + historyRealization.getRealization() + "' in '" + name + "'");
- }
+ public List<RealizationEntry> getRealizationEntries() {
+ return realizationEntries;
+ }
- if (realTimeRealizationInstance == null) {
- throw new IllegalArgumentException("Didn't find realization '" + realTimeRealization.getType() + "'" + " with name '" + realTimeRealization.getRealization() + "' in '" + name + "'");
+ private void init() {
+ if (initiated == true)
+ return;
+
+ synchronized (this) {
+ if (initiated == true)
+ return;
+
+ if (realizationEntries == null || realizationEntries.size() == 0)
+ throw new IllegalArgumentException();
+
+ RealizationRegistry registry = RealizationRegistry.getInstance(config);
+ List<IRealization> realizationList = Lists.newArrayList();
+ for (int i = 0; i < realizationEntries.size(); i++) {
+ IRealization realization = registry.getRealization(realizationEntries.get(i).getType(), realizationEntries.get(i).getRealization());
+ if (realization == null) {
+ logger.error("Realization '" + realization.getName() + " is not found, remove from Hybrid '" + this.getName() + "'");
+ continue;
+ }
+ if (realization.isReady() == false) {
+ logger.error("Realization '" + realization.getName() + " is disabled, remove from Hybrid '" + this.getName() + "'");
+ continue;
+ }
+ realizationList.add(realization);
+ }
+
+ LinkedHashSet<TblColRef> columns = new LinkedHashSet<TblColRef>();
+ LinkedHashSet<TblColRef> dimensions = new LinkedHashSet<TblColRef>();
+ LinkedHashSet<MeasureDesc> measures = new LinkedHashSet<MeasureDesc>();
+ dateRangeStart = 0;
+ dateRangeEnd = Long.MAX_VALUE;
+ for (IRealization realization : realizationList) {
+ columns.addAll(realization.getAllColumns());
+ dimensions.addAll(realization.getAllDimensions());
+ measures.addAll(realization.getMeasures());
+
+ if (realization.isReady())
+ isReady = true;
+
+ if (dateRangeStart == 0 || realization.getDateRangeStart() < dateRangeStart)
+ dateRangeStart = realization.getDateRangeStart();
+
+ if (dateRangeStart == Long.MAX_VALUE || realization.getDateRangeEnd() > dateRangeEnd)
+ dateRangeEnd = realization.getDateRangeEnd();
+ }
+
+ allDimensions = Lists.newArrayList(dimensions);
+ allColumns = Lists.newArrayList(columns);
+ allMeasures = Lists.newArrayList(measures);
+
+ Collections.sort(realizationList, new Comparator<IRealization>() {
+ @Override
+ public int compare(IRealization o1, IRealization o2) {
+
+ long i1 = o1.getDateRangeStart();
+ long i2 = o2.getDateRangeStart();
+ long comp = i1 - i2;
+ if (comp != 0) {
+ return comp > 0 ? 1 : -1;
+ }
+
+ i1 = o1.getDateRangeEnd();
+ i2 = o2.getDateRangeEnd();
+ comp = i1 - i2;
+ if (comp != 0) {
+ return comp > 0 ? 1 : -1;
+ }
+
+ return 0;
+ }
+ });
+
+ this.realizations = realizationList.toArray(new IRealization[realizationList.size()]);
+ initiated = true;
}
-
}
@Override
public boolean isCapable(SQLDigest digest) {
- return getHistoryRealizationInstance().isCapable(digest) && getRealTimeRealizationInstance().isCapable(digest);
+ for (IRealization realization : getRealizations()) {
+ if (realization.isCapable(digest))
+ return true;
+ }
+ return false;
}
@Override
public int getCost(SQLDigest digest) {
- return historyRealizationInstance.getCost(digest);
- //return Math.min(historyRealizationInstance.getCost(digest), realTimeRealizationInstance.getCost(digest));
+ return cost;
}
@Override
@@ -72,23 +151,30 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
@Override
+ public DataModelDesc getDataModelDesc() {
+ return this.getLatestRealization().getDataModelDesc();
+ }
+
+ @Override
public String getFactTable() {
- return getHistoryRealizationInstance().getFactTable();
+ return getRealizations()[0].getFactTable();
}
@Override
public List<TblColRef> getAllColumns() {
- return getHistoryRealizationInstance().getAllColumns();
+ init();
+ return allColumns;
}
@Override
public List<MeasureDesc> getMeasures() {
- return getHistoryRealizationInstance().getMeasures();
+ init();
+ return allMeasures;
}
@Override
public boolean isReady() {
- return historyRealizationInstance.isReady() || realTimeRealizationInstance.isReady();
+ return isReady;
}
@Override
@@ -128,61 +214,44 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
this.config = config;
}
- public RealizationEntry getHistoryRealization() {
- return historyRealization;
- }
-
- public RealizationEntry getRealTimeRealization() {
- return realTimeRealization;
- }
-
- public IRealization getHistoryRealizationInstance() {
- if (historyRealizationInstance == null) {
- this.init();
- }
- return historyRealizationInstance;
- }
-
- public IRealization getRealTimeRealizationInstance() {
- if (realTimeRealizationInstance == null) {
- this.init();
- }
- return realTimeRealizationInstance;
- }
-
@Override
public long getDateRangeStart() {
- return Math.min(getHistoryRealizationInstance().getDateRangeStart(), getRealTimeRealizationInstance().getDateRangeStart());
+ return dateRangeStart;
}
@Override
public long getDateRangeEnd() {
- return Math.max(getHistoryRealizationInstance().getDateRangeEnd(), getRealTimeRealizationInstance().getDateRangeEnd()) +1;
+ return dateRangeEnd;
}
-
-
- public DataModelDesc getDataModelDesc(){
- if (getHistoryRealizationInstance() instanceof CubeInstance) {
- return ((CubeInstance) historyRealizationInstance).getDescriptor().getModel();
- }
-
- return ((IIInstance) historyRealizationInstance).getDescriptor().getModel();
+ @Override
+ public String getModelName() {
+ return this.getLatestRealization().getModelName();
}
- @Override
- public String getModelName() {
- if (getHistoryRealizationInstance() instanceof CubeInstance) {
- return ((CubeInstance) historyRealizationInstance).getDescriptor().getModelName();
- }
+ @Override
+ public List<TblColRef> getAllDimensions() {
+ init();
+ return allDimensions;
+ }
- return ((IIInstance) historyRealizationInstance).getDescriptor().getModelName();
+ public IRealization[] getRealizations() {
+ init();
+ return realizations;
}
- @Override
- public List<TblColRef> getAllDimensions(){
+ public static String concatResourcePath(String hybridName) {
+ return ResourceStore.HYBRID_RESOURCE_ROOT + "/" + hybridName + ".json";
+ }
- return this.getHistoryRealizationInstance().getAllDimensions();
+ public void setCost(int cost) {
+ this.cost = cost;
}
+ public IRealization getLatestRealization() {
+ if (realizations.length > 0) {
+ return realizations[realizations.length - 1];
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index df3fae3..75af89f 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -81,6 +81,22 @@ public class HybridManager implements IRealizationProvider {
logger.debug("Loaded " + paths.size() + " Hybrid(s)");
}
+ public void reloadHybridInstanceByChild(RealizationType type, String realizationName) throws IOException {
+ for (HybridInstance hybridInstance : hybridMap.values()) {
+ boolean includes = false;
+ for (IRealization realization : hybridInstance.getRealizations()) {
+ if (realization.getType() == type && realization.getName().equalsIgnoreCase(realizationName)) {
+ includes = true;
+ break;
+ }
+ }
+
+ if (includes == true)
+ loadHybridInstance(HybridInstance.concatResourcePath(hybridInstance.getName()));
+ }
+ }
+
+
private synchronized HybridInstance loadHybridInstance(String path) throws IOException {
ResourceStore store = getStore();
@@ -89,14 +105,13 @@ public class HybridManager implements IRealizationProvider {
hybridInstance = store.getResource(path, HybridInstance.class, HYBRID_SERIALIZER);
hybridInstance.setConfig(config);
+ if (hybridInstance.getRealizationEntries() == null || hybridInstance.getRealizationEntries().size() == 0) {
+ throw new IllegalStateException("HybridInstance must have realization entries, " + path);
+ }
+
if (StringUtils.isBlank(hybridInstance.getName()))
throw new IllegalStateException("HybridInstance name must not be blank, at " + path);
- if (hybridInstance.getHistoryRealization() == null || hybridInstance.getRealTimeRealization() == null) {
-
- throw new IllegalStateException("HybridInstance must have both historyRealization and realTimeRealization set, at " + path);
- }
-
final String name = hybridInstance.getName();
hybridMap.putLocal(name, hybridInstance);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index b7219b5..3698688 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,14 +1,8 @@
package org.apache.kylin.storage.hybrid;
-import com.google.common.base.Function;
import com.google.common.collect.Lists;
-import com.google.common.collect.Ranges;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.realization.SQLDigestUtil;
import org.apache.kylin.metadata.tuple.CompoundTupleIterator;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.storage.IStorageEngine;
@@ -16,54 +10,35 @@ import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.StorageEngineFactory;
import org.apache.kylin.storage.tuple.TupleInfo;
-import javax.annotation.Nullable;
+import java.util.List;
/**
*/
public class HybridStorageEngine implements IStorageEngine {
- private HybridInstance hybridInstance;
- private IStorageEngine historicalStorageEngine;
- private IStorageEngine realtimeStorageEngine;
+ private IRealization[] realizations;
+ private IStorageEngine[] storageEngines;
public HybridStorageEngine(HybridInstance hybridInstance) {
- this.hybridInstance = hybridInstance;
- this.historicalStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getHistoryRealizationInstance());
- this.realtimeStorageEngine = StorageEngineFactory.getStorageEngine(this.hybridInstance.getRealTimeRealizationInstance());
+ this.realizations = hybridInstance.getRealizations();
+ storageEngines = new IStorageEngine[realizations.length];
+ for (int i = 0; i < realizations.length; i++) {
+ storageEngines[i] = StorageEngineFactory.getStorageEngine(realizations[i]);
+ }
}
@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
-
- // search the historic realization
- ITupleIterator historicalDataIterator = this.historicalStorageEngine.search(context, sqlDigest, returnTupleInfo);
-
- String modelName = hybridInstance.getModelName();
- MetadataManager metaMgr = getMetadataManager();
- DataModelDesc modelDesc = metaMgr.getDataModelDesc(modelName);
-
- // if the model isn't partitioned, only query the history
- if (modelDesc.getPartitionDesc() == null || modelDesc.getPartitionDesc().getPartitionDateColumnRef() == null)
- return historicalDataIterator;
-
- TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
-
- ITupleIterator realtimeDataIterator = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef, //
- Ranges.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),//
- new Function<Void, ITupleIterator>() {
- @Nullable
- @Override
- public ITupleIterator apply(@Nullable Void input) {
- return realtimeStorageEngine.search(context, sqlDigest, returnTupleInfo);
- }
- });
-
- // combine history and real-time tuple iterator
- return new CompoundTupleIterator(Lists.newArrayList(historicalDataIterator, realtimeDataIterator));
+ List<ITupleIterator> tupleIterators = Lists.newArrayList();
+ for (int i = 0; i < realizations.length; i++) {
+ if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest)) {
+ ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest, returnTupleInfo);
+ tupleIterators.add(dataIterator);
+ }
+ }
+ // combine tuple iterator
+ return new CompoundTupleIterator(tupleIterators);
}
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f8c2283d/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
index 3ad7ca6..eaa1db5 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -26,19 +26,18 @@ public class HybridManagerTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() throws Exception {
- HybridInstance cube = getHybridManager().getHybridInstance("test_kylin_hybrid_left_join");
- cube.init();
- System.out.println(JsonUtil.writeValueAsIndentString(cube));
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_ready");
+ System.out.println(JsonUtil.writeValueAsIndentString(hybridInstance));
- IRealization history = cube.getHistoryRealizationInstance();
- IRealization realTime = cube.getRealTimeRealizationInstance();
+ IRealization[] realizations = hybridInstance.getRealizations();
+ Assert.assertEquals(realizations.length, 2);
- Assert.assertTrue(history instanceof CubeInstance);
- Assert.assertTrue(realTime instanceof IIInstance);
+ IRealization lastReal = hybridInstance.getLatestRealization();
+ Assert.assertTrue(lastReal instanceof CubeInstance);
+ Assert.assertEquals(lastReal.getName(), "test_kylin_cube_with_slr_ready_2_segments");
}
-
public HybridManager getHybridManager() {
return HybridManager.getInstance(getTestConfig());
}