You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/12/02 06:51:38 UTC
spark git commit: [SPARK-11949][SQL] Check bitmasks to set nullable
property
Repository: spark
Updated Branches:
refs/heads/master 8a75a3049 -> 0f37d1d7e
[SPARK-11949][SQL] Check bitmasks to set nullable property
Following up #10038.
We can use bitmasks to determine which grouping expressions need to be set as nullable.
cc yhuai
Author: Liang-Chi Hsieh <vi...@appier.com>
Closes #10067 from viirya/fix-cube-following.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0f37d1d7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0f37d1d7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0f37d1d7
Branch: refs/heads/master
Commit: 0f37d1d7ed7f6e34f98f2a3c274918de29e7a1d7
Parents: 8a75a30
Author: Liang-Chi Hsieh <vi...@appier.com>
Authored: Tue Dec 1 21:51:33 2015 -0800
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Dec 1 21:51:33 2015 -0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/analysis/Analyzer.scala | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0f37d1d7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 765327c..d3163dc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -224,10 +224,15 @@ class Analyzer(
case other => Alias(other, other.toString)()
}
- // TODO: We need to use bitmasks to determine which grouping expressions need to be
- // set as nullable. For example, if we have GROUPING SETS ((a,b), a), we do not need
- // to change the nullability of a.
- val attributeMap = groupByAliases.map(a => (a -> a.toAttribute.withNullability(true))).toMap
+ val nonNullBitmask = x.bitmasks.reduce(_ & _)
+
+ val attributeMap = groupByAliases.zipWithIndex.map { case (a, idx) =>
+ if ((nonNullBitmask & 1 << idx) == 0) {
+ (a -> a.toAttribute.withNullability(true))
+ } else {
+ (a -> a.toAttribute)
+ }
+ }.toMap
val aggregations: Seq[NamedExpression] = x.aggregations.map {
// If an expression is an aggregate (contains a AggregateExpression) then we dont change
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org