You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/27 06:51:50 UTC
carbondata git commit: [CARBONDATA-3131] Update the requested columns
to the Scan
Repository: carbondata
Updated Branches:
refs/heads/master 877efe1a4 -> dbb7ca1b1
[CARBONDATA-3131] Update the requested columns to the Scan
Get the updated list of requested columns for Scan task
This closes #2952
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dbb7ca1b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dbb7ca1b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dbb7ca1b
Branch: refs/heads/master
Commit: dbb7ca1b17ca647228a5ae04db3e5870af5ad5c8
Parents: 877efe1
Author: dhatchayani <dh...@gmail.com>
Authored: Tue Nov 27 10:13:33 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Nov 27 12:21:40 2018 +0530
----------------------------------------------------------------------
.../execution/strategy/CarbonLateDecodeStrategy.scala | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dbb7ca1b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 6993ecc..a2c4bd0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -390,8 +390,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
}
// Don't request columns that are only referenced by pushed filters.
val requestedColumns =
- (projectsAttr.to[mutable.LinkedHashSet] ++ filterSet -- handledSet)
- .map(relation.attributeMap).toSeq ++ newProjectList
+ getRequestedColumns(relation, projectsAttr, filterSet, handledSet, newProjectList)
var updateRequestedColumns =
if (!vectorPushRowFilters && !implictsExisted && !hasDictionaryFilterCols
@@ -446,6 +445,15 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
}
}
+ protected def getRequestedColumns(relation: LogicalRelation,
+ projectsAttr: Seq[Attribute],
+ filterSet: AttributeSet,
+ handledSet: AttributeSet,
+ newProjectList: Seq[Attribute]) = {
+ (projectsAttr.to[mutable.LinkedHashSet] ++ filterSet -- handledSet)
+ .map(relation.attributeMap).toSeq ++ newProjectList
+ }
+
private def getDataSourceScan(relation: LogicalRelation,
output: Seq[Attribute],
partitions: Seq[PartitionSpec],