You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/13 10:25:10 UTC
[kylin] 03/25: KYLIN-5332 Query select star expose column from dataflow
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 82516523250251fbf6ed6a1570eb3789b54b7e13
Author: songzhxlh-max <59...@users.noreply.github.com>
AuthorDate: Fri Oct 21 15:07:28 2022 +0800
KYLIN-5332 Query select star expose column from dataflow
---
.../metadata/cube/cuboid/NLayoutCandidate.java | 4 ++
.../metadata/cube/cuboid/NQueryLayoutChooser.java | 7 +--
.../cube/model/NDataflowCapabilityChecker.java | 1 +
.../org/apache/kylin/query/routing/Candidate.java | 62 ++++++++++++++--------
.../kylin/query/routing/RealizationChooser.java | 14 ++++-
5 files changed, 62 insertions(+), 26 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java
index a7ceda70a2..caef168503 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NLayoutCandidate.java
@@ -42,6 +42,10 @@ public class NLayoutCandidate implements IRealizationCandidate {
@Setter
private double cost;
+ @Getter
+ @Setter
+ private int layoutUnmatchedColsSize;
+
@Setter
@Getter
private CapabilityResult capabilityResult;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java
index 0b32223e33..ccada17bd7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/NQueryLayoutChooser.java
@@ -110,7 +110,6 @@ public class NQueryLayoutChooser {
}
val projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
.getProject(dataflow.getProject());
- double influenceFactor = 1.0;
for (NDataLayout dataLayout : commonLayouts) {
log.trace("Matching layout {}", dataLayout);
CapabilityResult tempResult = new CapabilityResult();
@@ -119,18 +118,20 @@ public class NQueryLayoutChooser {
LayoutEntity layout = indexPlan.getLayoutEntity(dataLayout.getLayoutId());
log.trace("Matching indexEntity {}", indexEntity);
+ NLayoutCandidate candidate = new NLayoutCandidate(layout);
var matchResult = tableIndexMatcher.match(layout);
+ double influenceFactor = 1.0;
if (!matchResult.isMatched()) {
matchResult = aggIndexMatcher.match(layout);
} else if (projectInstance.getConfig().useTableIndexAnswerSelectStarEnabled()) {
- influenceFactor += influenceFactor + tableIndexMatcher.getLayoutUnmatchedColsSize();
+ influenceFactor += tableIndexMatcher.getLayoutUnmatchedColsSize();
+ candidate.setLayoutUnmatchedColsSize(tableIndexMatcher.getLayoutUnmatchedColsSize());
}
if (!matchResult.isMatched()) {
log.trace("Matching failed");
continue;
}
- NLayoutCandidate candidate = new NLayoutCandidate(layout);
tempResult.influences = matchResult.getInfluences();
candidate.setCost(dataLayout.getRows() * (tempResult.influences.size() + influenceFactor));
if (!matchResult.getNeedDerive().isEmpty()) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java
index 0968253dbe..34d3a016f8 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/NDataflowCapabilityChecker.java
@@ -85,6 +85,7 @@ public class NDataflowCapabilityChecker {
if (candidateAndInfluence != null) {
chosenCandidate = candidateAndInfluence;
result.influences.addAll(candidateAndInfluence.getCapabilityResult().influences);
+ result.setLayoutUnmatchedColsSize(candidateAndInfluence.getLayoutUnmatchedColsSize());
logger.info("Matched layout {} snapshot in dataflow {} ", chosenCandidate, dataflow);
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java
index c8eb9a2247..06ad7c2b65 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -39,6 +39,7 @@ import lombok.Setter;
public class Candidate {
public static final CandidateComparator COMPARATOR = new CandidateComparator();
+ public static final CandidateTableIndexComparator COMPARATOR_TABLE_INDEX = new CandidateTableIndexComparator();
// ============================================================================
@@ -109,36 +110,53 @@ public class Candidate {
@Override
public int compare(Candidate c1, Candidate c2) {
- IRealization real1 = c1.getRealization();
- IRealization real2 = c2.getRealization();
-
- if (QueryContext.current().getModelPriorities().length > 0) {
-
- Map<String, Integer> priorities = new HashMap<>();
- for (int i = 0; i < QueryContext.current().getModelPriorities().length; i++) {
- priorities.put(QueryContext.current().getModelPriorities()[i], i);
- }
-
- int comp = priorities.getOrDefault(real1.getModel().getAlias().toUpperCase(Locale.ROOT),
- Integer.MAX_VALUE)
- - priorities.getOrDefault(real2.getModel().getAlias().toUpperCase(Locale.ROOT),
- Integer.MAX_VALUE);
- if (comp != 0) {
- return comp;
- }
+ return compareCandidate(c1, c2);
+ }
+ }
+
+ public static class CandidateTableIndexComparator implements Comparator<Candidate> {
+
+ @Override
+ public int compare(Candidate c1, Candidate c2) {
+ CapabilityResult capabilityResult1 = c1.getCapability();
+ CapabilityResult capabilityResult2 = c2.getCapability();
+ if (capabilityResult1.getLayoutUnmatchedColsSize() != capabilityResult2.getLayoutUnmatchedColsSize()) {
+ return capabilityResult1.getLayoutUnmatchedColsSize() - capabilityResult2.getLayoutUnmatchedColsSize();
}
+ return compareCandidate(c1, c2);
+ }
+ }
- int comp = real1.getCost() - real2.getCost();
- if (comp != 0) {
- return comp;
+ private static int compareCandidate(Candidate c1, Candidate c2) {
+ IRealization real1 = c1.getRealization();
+ IRealization real2 = c2.getRealization();
+
+ if (QueryContext.current().getModelPriorities().length > 0) {
+
+ Map<String, Integer> priorities = new HashMap<>();
+ for (int i = 0; i < QueryContext.current().getModelPriorities().length; i++) {
+ priorities.put(QueryContext.current().getModelPriorities()[i], i);
}
- comp = Double.compare(c1.capability.cost, c2.capability.cost);
+ int comp = priorities.getOrDefault(real1.getModel().getAlias().toUpperCase(Locale.ROOT),
+ Integer.MAX_VALUE)
+ - priorities.getOrDefault(real2.getModel().getAlias().toUpperCase(Locale.ROOT),
+ Integer.MAX_VALUE);
if (comp != 0) {
return comp;
}
+ }
+
+ int comp = real1.getCost() - real2.getCost();
+ if (comp != 0) {
+ return comp;
+ }
- return real1.getModel().getId().compareTo(real2.getModel().getId());
+ comp = Double.compare(c1.capability.cost, c2.capability.cost);
+ if (comp != 0) {
+ return comp;
}
+
+ return real1.getModel().getId().compareTo(real2.getModel().getId());
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index b27d486e55..284a2fc190 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -91,6 +91,7 @@ import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectLoader;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
@@ -243,7 +244,7 @@ public class RealizationChooser {
}
// Step 3. find the lowest-cost candidate
- candidates.sort(Candidate.COMPARATOR);
+ sortCandidate(context, candidates);
logger.trace("Cost Sorted Realizations {}", candidates);
if (!candidates.isEmpty()) {
Candidate selectedCandidate = candidates.get(0);
@@ -273,6 +274,17 @@ public class RealizationChooser {
throw new NoRealizationFoundException("No realization found for " + toErrorMsg(context));
}
+ private static void sortCandidate(OLAPContext context, List<Candidate> candidates) {
+ ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
+ .getProject(context.olapSchema.getProjectName());
+ if (projectInstance.getConfig().useTableIndexAnswerSelectStarEnabled()
+ && context.getSQLDigest().isRawQuery) {
+ candidates.sort(Candidate.COMPARATOR_TABLE_INDEX);
+ } else {
+ candidates.sort(Candidate.COMPARATOR);
+ }
+ }
+
private static void checkNoRealizationWithStreaming(OLAPContext context) {
String projectName = context.olapSchema.getProjectName();
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();