spark git commit: [SPARK-13540][SQL] Supports using nested classes within Scala objects as Dataset element type

[li...@apache.org]

Repository: spark
Updated Branches:
  refs/heads/master ac5c63528 -> 916fc34f9


[SPARK-13540][SQL] Supports using nested classes within Scala objects as Dataset element type

## What changes were proposed in this pull request?

Nested classes defined within Scala objects are translated into Java static nested classes. Unlike inner classes, they don't need outer scopes. But the analyzer still thinks that an outer scope is required.

This PR fixes this issue simply by checking whether a nested class is static before looking up its outer scope.

## How was this patch tested?

A test case is added to `DatasetSuite`. It checks contents of a Dataset whose element type is a nested class declared in a Scala object.

Author: Cheng Lian <li...@databricks.com>

Closes #11421 from liancheng/spark-13540-object-as-outer-scope.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/916fc34f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/916fc34f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/916fc34f

Branch: refs/heads/master
Commit: 916fc34f98dd731f607d9b3ed657bad6cc30df2c
Parents: ac5c635
Author: Cheng Lian <li...@databricks.com>
Authored: Tue Mar 1 01:07:45 2016 +0800
Committer: Cheng Lian <li...@databricks.com>
Committed: Tue Mar 1 01:07:45 2016 +0800

----------------------------------------------------------------------
 .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 10 +++++++++-
 .../test/scala/org/apache/spark/sql/DatasetSuite.scala    | 10 ++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/916fc34f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 23e4709..876aa0e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
+import java.lang.reflect.Modifier
+
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.sql.AnalysisException
@@ -559,7 +561,13 @@ class Analyzer(
       }
 
       resolveExpression(unbound, LocalRelation(attributes), throws = true) transform {
-        case n: NewInstance if n.outerPointer.isEmpty && n.cls.isMemberClass =>
+        case n: NewInstance
+          // If this is an inner class of another class, register the outer object in `OuterScopes`.
+          // Note that static inner classes (e.g., inner classes within Scala objects) don't need
+          // outer pointer registration.
+          if n.outerPointer.isEmpty &&
+             n.cls.isMemberClass &&
+             !Modifier.isStatic(n.cls.getModifiers) =>
           val outer = OuterScopes.outerScopes.get(n.cls.getDeclaringClass.getName)
           if (outer == null) {
             throw new AnalysisException(

http://git-wip-us.apache.org/repos/asf/spark/blob/916fc34f/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 14fc37b..33df637 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -621,12 +621,22 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
       ds.filter(_ => true),
       Some(1), Some(2), Some(3))
   }
+
+  test("SPARK-13540 Dataset of nested class defined in Scala object") {
+    checkAnswer(
+      Seq(OuterObject.InnerClass("foo")).toDS(),
+      OuterObject.InnerClass("foo"))
+  }
 }
 
 class OuterClass extends Serializable {
   case class InnerClass(a: String)
 }
 
+object OuterObject {
+  case class InnerClass(a: String)
+}
+
 case class ClassData(a: String, b: Int)
 case class ClassData2(c: String, d: Int)
 case class ClassNullableData(a: String, b: Integer)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org