You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/02/14 05:54:40 UTC
[kylin] 11/33: KYLIN-5430 skip shard pruning for in expr
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1e08d51b758c5f8975a743c14fecc42c39931ba8
Author: Dorris Zhang <ru...@kyligence.io>
AuthorDate: Mon Nov 28 13:23:01 2022 +0800
KYLIN-5430 skip shard pruning for in expr
---
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../apache/spark/sql/execution/datasource/FilePruner.scala | 14 +++++++++++---
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 53ca8c6e00..4087654ce8 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3741,4 +3741,8 @@ public abstract class KylinConfigBase implements Serializable {
public int getProjectMergeRuleBloatThreshold() {
return Integer.parseInt(getOptional("kylin.query.project-merge-bloat-threshold", "0"));
}
+
+ public boolean skipShardPruningForInExpr() {
+ return Boolean.parseBoolean(getOptional("kylin.query.skip-shard-pruning-for-in", FALSE));
+ }
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
index 8e8c046410..b658a0bd7e 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/execution/datasource/FilePruner.scala
@@ -455,8 +455,14 @@ class FilePruner(val session: SparkSession,
def getShardSetFromIterable(attr: Attribute, iter: Iterable[Any]): BitSet = {
val matchedShards = new BitSet(numShards)
- iter.map(v => getShardNumber(attr, v))
- .foreach(shardNum => matchedShards.set(shardNum))
+ val prj = options.getOrElse("project", sys.error("project option is required"))
+ val skipShardPruning = NProjectManager.getProjectConfig(prj).skipShardPruningForInExpr && iter.size > 256
+ if (skipShardPruning) {
+ matchedShards.setUntil(matchedShards.capacity)
+ } else {
+ iter.map(v => getShardNumber(attr, v))
+ .foreach(shardNum => matchedShards.set(shardNum))
+ }
matchedShards
}
@@ -738,7 +744,9 @@ case class SegDimFilters(dimRange: java.util.Map[String, DimensionRangeInfo], di
* blocks are always non-empty.
*/
- def escapeQuote(colName: String): String = {s"${colName.replace("`", "")}"}
+ def escapeQuote(colName: String): String = {
+ s"${colName.replace("`", "")}"
+ }
def foldFilter(filter: Filter): Filter = {
filter match {