You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/07/15 23:01:54 UTC
git commit: [SPARK-2483][SQL] Fix parsing of repeated,
nested data access.
Repository: spark
Updated Branches:
refs/heads/master a21f9a754 -> 0f98ef1a2
[SPARK-2483][SQL] Fix parsing of repeated, nested data access.
Author: Michael Armbrust <mi...@databricks.com>
Closes #1411 from marmbrus/nestedRepeated and squashes the following commits:
044fa09 [Michael Armbrust] Fix parsing of repeated, nested data access.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0f98ef1a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0f98ef1a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0f98ef1a
Branch: refs/heads/master
Commit: 0f98ef1a2c9ecf328f6c5918808fa5ca486e8afd
Parents: a21f9a7
Author: Michael Armbrust <mi...@databricks.com>
Authored: Tue Jul 15 14:01:48 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Tue Jul 15 14:01:48 2014 -0700
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/hive/HiveQl.scala | 5 +----
.../spark/sql/hive/execution/HiveResolutionSuite.scala | 10 ++++++++--
2 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0f98ef1a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index b70104d..56aa27a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -870,10 +870,7 @@ private[hive] object HiveQl {
nodeToExpr(qualifier) match {
case UnresolvedAttribute(qualifierName) =>
UnresolvedAttribute(qualifierName + "." + cleanIdentifier(attr))
- // The precidence for . seems to be wrong, so [] binds tighter an we need to go inside to
- // find the underlying attribute references.
- case GetItem(UnresolvedAttribute(qualifierName), ordinal) =>
- GetItem(UnresolvedAttribute(qualifierName + "." + cleanIdentifier(attr)), ordinal)
+ case other => GetField(other, attr)
}
/* Stars (*) */
http://git-wip-us.apache.org/repos/asf/spark/blob/0f98ef1a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
index 67594b5..fb03db1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.hive.test.TestHive._
-case class Data(a: Int, B: Int, n: Nested)
+case class Data(a: Int, B: Int, n: Nested, nestedArray: Seq[Nested])
case class Nested(a: Int, B: Int)
/**
@@ -53,12 +53,18 @@ class HiveResolutionSuite extends HiveComparisonTest {
test("case insensitivity with scala reflection") {
// Test resolution with Scala Reflection
- TestHive.sparkContext.parallelize(Data(1, 2, Nested(1,2)) :: Nil)
+ TestHive.sparkContext.parallelize(Data(1, 2, Nested(1,2), Seq(Nested(1,2))) :: Nil)
.registerAsTable("caseSensitivityTest")
hql("SELECT a, b, A, B, n.a, n.b, n.A, n.B FROM caseSensitivityTest")
}
+ test("nested repeated resolution") {
+ TestHive.sparkContext.parallelize(Data(1, 2, Nested(1,2), Seq(Nested(1,2))) :: Nil)
+ .registerAsTable("nestedRepeatedTest")
+ assert(hql("SELECT nestedArray[0].a FROM nestedRepeatedTest").collect().head(0) === 1)
+ }
+
/**
* Negative examples. Currently only left here for documentation purposes.
* TODO(marmbrus): Test that catalyst fails on these queries.