You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/04/06 07:10:14 UTC
[kylin] branch master updated: [KYLIN-4794] Make it possible to
force hit a cube set for sqls with cube join
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new e44aee2 [KYLIN-4794] Make it possible to force hit a cube set for sqls with cube join
e44aee2 is described below
commit e44aee2eb6f8e8686f6e4edc3de1a90493d00288
Author: yangjiang <ya...@ebay.com>
AuthorDate: Mon Nov 16 18:52:22 2020 +0800
[KYLIN-4794] Make it possible to force hit a cube set for sqls with cube join
---
.../query/enumerator/LookupTableEnumerator.java | 23 ++++++++++++++--
.../apache/kylin/query/routing/QueryRouter.java | 31 +++++++++++++++++-----
2 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index a4f28a8..ad2e20c 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -20,10 +20,15 @@ package org.apache.kylin.query.enumerator;
import java.io.IOException;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
+import java.util.Set;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.DimensionDesc;
@@ -60,8 +65,22 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
List<RealizationEntry> realizationEntries = project.getRealizationEntries();
String lookupTableName = olapContext.firstTableScan.getTableName();
CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
- cube = cubeMgr.findLatestSnapshot(realizationEntries, lookupTableName, cube);
- olapContext.realization = cube;
+
+ // Make force hit cube in lookup table
+ String forceHitCubeName = BackdoorToggles.getForceHitCube();
+ if (!StringUtil.isEmpty(forceHitCubeName)) {
+ String forceHitCubeNameLower = forceHitCubeName.toLowerCase(Locale.ROOT);
+ String[] forceHitCubeNames = forceHitCubeNameLower.split(",");
+ final Set<String> forceHitCubeNameSet = new HashSet<String>(Arrays.asList(forceHitCubeNames));
+ cube = cubeMgr.findLatestSnapshot(
+ (List<RealizationEntry>) realizationEntries.stream()
+ .filter(x -> forceHitCubeNameSet.contains(x.getRealization().toLowerCase(Locale.ROOT))),
+ lookupTableName, cube);
+ olapContext.realization = cube;
+ } else {
+ cube = cubeMgr.findLatestSnapshot(realizationEntries, lookupTableName, cube);
+ olapContext.realization = cube;
+ }
} else if (olapContext.realization instanceof HybridInstance) {
final HybridInstance hybridInstance = (HybridInstance) olapContext.realization;
final IRealization latestRealization = hybridInstance.getLatestRealization();
diff --git a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
index b8ee98b..8109430 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/QueryRouter.java
@@ -18,11 +18,15 @@
package org.apache.kylin.query.routing;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
@@ -50,15 +54,30 @@ public class QueryRouter {
String projectName = olapContext.olapSchema.getProjectName();
SQLDigest sqlDigest = olapContext.getSQLDigest();
- List<Candidate> candidates = Lists.newArrayListWithCapacity(realizations.size());
+ String forceHitCubeName = BackdoorToggles.getForceHitCube();
+ Set<String> forceHitCubeNameSet = new HashSet<String>();
+ if (!StringUtil.isEmpty(forceHitCubeName)) {
+ String forceHitCubeNameLower = forceHitCubeName.toLowerCase(Locale.ROOT);
+ String[] forceHitCubeNames = forceHitCubeNameLower.split(",");
+ forceHitCubeNameSet = new HashSet<String>(Arrays.asList(forceHitCubeNames));
+ }
+
+ List<Candidate> candidates = Lists.newArrayList();
for (IRealization real : realizations) {
- if (real.isReady())
- candidates.add(new Candidate(real, sqlDigest));
- if (BackdoorToggles.getForceHitCube() != null && BackdoorToggles.getForceHitCube().equalsIgnoreCase(real.getName())) {
- logger.info("Force choose {} as selected cube for specific purpose.", real.getName());
- candidates = Lists.newArrayListWithCapacity(1);
+ if (!forceHitCubeNameSet.isEmpty()) {
+ if (!forceHitCubeNameSet.contains(real.getName().toLowerCase(Locale.ROOT))) {
+ continue;
+ }
+ if (!real.isReady()) {
+ throw new RuntimeException(
+ "Realization " + real.getName() + " is not ready and should not be force hit");
+ }
candidates.add(new Candidate(real, sqlDigest));
break;
+ } else {
+ if (real.isReady()) {
+ candidates.add(new Candidate(real, sqlDigest));
+ }
}
}