You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/01/21 07:41:42 UTC
spark git commit: [SPARK-4959][SQL] Attributes are case sensitive
when using a select query from a projection(Backport to Spark-1.2)
Repository: spark
Updated Branches:
refs/heads/branch-1.2 692dc5b66 -> 92c238c19
[SPARK-4959][SQL] Attributes are case sensitive when using a select query from a projection(Backport to Spark-1.2)
This is a follow up of #3796 , which can not be merged back to Spark-1.2. Manually merge it.
Author: Cheng Hao <ha...@intel.com>
Closes #4013 from chenghao-intel/spark_4959_backport and squashes the following commits:
1f6c93d [Cheng Hao] backport to Spark-1.2
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/92c238c1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/92c238c1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/92c238c1
Branch: refs/heads/branch-1.2
Commit: 92c238c192287ae12c3028b8ee3d9b5d9a65c72a
Parents: 692dc5b
Author: Cheng Hao <ha...@intel.com>
Authored: Tue Jan 20 22:41:38 2015 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Jan 20 22:41:38 2015 -0800
----------------------------------------------------------------------
.../spark/sql/catalyst/optimizer/Optimizer.scala | 8 ++++----
.../sql/hive/execution/HiveTableScanSuite.scala | 15 +++++++++++++--
2 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/92c238c1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index f164a6c..c2aafb9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -142,16 +142,16 @@ object ColumnPruning extends Rule[LogicalPlan] {
case Project(projectList1, Project(projectList2, child)) =>
// Create a map of Aliases to their values from the child projection.
// e.g., 'SELECT ... FROM (SELECT a + b AS c, d ...)' produces Map(c -> Alias(a + b, c)).
- val aliasMap = projectList2.collect {
- case a @ Alias(e, _) => (a.toAttribute: Expression, a)
- }.toMap
+ val aliasMap = AttributeMap(projectList2.collect {
+ case a @ Alias(e, _) => (a.toAttribute, a)
+ })
// Substitute any attributes that are produced by the child projection, so that we safely
// eliminate it.
// e.g., 'SELECT c + 1 FROM (SELECT a + b AS C ...' produces 'SELECT a + b + 1 ...'
// TODO: Fix TransformBase to avoid the cast below.
val substitutedProjection = projectList1.map(_.transform {
- case a if aliasMap.contains(a) => aliasMap(a)
+ case a: Attribute if aliasMap.contains(a) => aliasMap(a)
}).asInstanceOf[Seq[NamedExpression]]
Project(substitutedProjection, child)
http://git-wip-us.apache.org/repos/asf/spark/blob/92c238c1/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
index 54c0f01..27f40a2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
@@ -18,7 +18,8 @@
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.hive.test.TestHive
-import org.apache.spark.sql.{Row, SchemaRDD}
+import org.apache.spark.sql.hive.test.TestHive._
+import org.apache.spark.sql.Row
import org.apache.spark.util.Utils
@@ -68,5 +69,15 @@ class HiveTableScanSuite extends HiveComparisonTest {
=== Array(Row(java.sql.Timestamp.valueOf("2014-12-11 00:00:00")),Row(null)))
TestHive.sql("DROP TABLE timestamp_query_null")
}
-
+
+ test("Spark-4959 Attributes are case sensitive when using a select query from a projection") {
+ sql("create table spark_4959 (col1 string)")
+ sql("""insert into table spark_4959 select "hi" from src limit 1""")
+ table("spark_4959").select(
+ 'col1.as('CaseSensitiveColName),
+ 'col1.as('CaseSensitiveColName2)).registerTempTable("spark_4959_2")
+
+ assert(sql("select CaseSensitiveColName from spark_4959_2").first() === Row("hi"))
+ assert(sql("select casesensitivecolname from spark_4959_2").first() === Row("hi"))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org