You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/04/12 04:36:10 UTC
spark git commit: [SQL][minor] move `resolveGetField` into a object
Repository: spark
Updated Branches:
refs/heads/master 6d4e854ff -> 5c2844c51
[SQL][minor] move `resolveGetField` into a object
The method `resolveGetField` isn't belong to `LogicalPlan` logically and didn't access any members of it.
Author: Wenchen Fan <cl...@outlook.com>
Closes #5435 from cloud-fan/tmp and squashes the following commits:
9a66c83 [Wenchen Fan] code clean up
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5c2844c5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5c2844c5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5c2844c5
Branch: refs/heads/master
Commit: 5c2844c51aca6a0da9251a3fd346a6f872cf17f8
Parents: 6d4e854
Author: Wenchen Fan <cl...@outlook.com>
Authored: Sat Apr 11 19:35:56 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Sat Apr 11 19:35:56 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../sql/catalyst/expressions/complexTypes.scala | 37 ++++++++++++++++++++
.../catalyst/plans/logical/LogicalPlan.scala | 35 +-----------------
3 files changed, 39 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5c2844c5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index b83f18a..fd1ceb1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -308,7 +308,7 @@ class Analyzer(
logDebug(s"Resolving $u to $result")
result
case UnresolvedGetField(child, fieldName) if child.resolved =>
- q.resolveGetField(child, fieldName, resolver)
+ GetField(child, fieldName, resolver)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5c2844c5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
index 3b2b921..fc1f696 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions
import scala.collection.Map
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.Resolver
import org.apache.spark.sql.types._
/**
@@ -81,6 +83,41 @@ trait GetField extends UnaryExpression {
def field: StructField
}
+object GetField {
+ /**
+ * Returns the resolved `GetField`, and report error if no desired field or over one
+ * desired fields are found.
+ */
+ def apply(
+ expr: Expression,
+ fieldName: String,
+ resolver: Resolver): GetField = {
+ def findField(fields: Array[StructField]): Int = {
+ val checkField = (f: StructField) => resolver(f.name, fieldName)
+ val ordinal = fields.indexWhere(checkField)
+ if (ordinal == -1) {
+ throw new AnalysisException(
+ s"No such struct field $fieldName in ${fields.map(_.name).mkString(", ")}")
+ } else if (fields.indexWhere(checkField, ordinal + 1) != -1) {
+ throw new AnalysisException(
+ s"Ambiguous reference to fields ${fields.filter(checkField).mkString(", ")}")
+ } else {
+ ordinal
+ }
+ }
+ expr.dataType match {
+ case StructType(fields) =>
+ val ordinal = findField(fields)
+ StructGetField(expr, fields(ordinal), ordinal)
+ case ArrayType(StructType(fields), containsNull) =>
+ val ordinal = findField(fields)
+ ArrayGetField(expr, fields(ordinal), ordinal, containsNull)
+ case otherType =>
+ throw new AnalysisException(s"GetField is not valid on fields of type $otherType")
+ }
+ }
+}
+
/**
* Returns the value of fields in the Struct `child`.
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/5c2844c5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index d8f5858..579a0fb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -210,7 +210,7 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
// For example, consider "a.b.c", where "a" is resolved to an existing attribute.
// Then this will add GetField("c", GetField("b", a)), and alias
// the final expression as "c".
- val fieldExprs = nestedFields.foldLeft(a: Expression)(resolveGetField(_, _, resolver))
+ val fieldExprs = nestedFields.foldLeft(a: Expression)(GetField(_, _, resolver))
val aliasName = nestedFields.last
Some(Alias(fieldExprs, aliasName)())
} catch {
@@ -229,39 +229,6 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
s"Reference '$name' is ambiguous, could be: $referenceNames.")
}
}
-
- /**
- * Returns the resolved `GetField`, and report error if no desired field or over one
- * desired fields are found.
- */
- def resolveGetField(
- expr: Expression,
- fieldName: String,
- resolver: Resolver): Expression = {
- def findField(fields: Array[StructField]): Int = {
- val checkField = (f: StructField) => resolver(f.name, fieldName)
- val ordinal = fields.indexWhere(checkField)
- if (ordinal == -1) {
- throw new AnalysisException(
- s"No such struct field $fieldName in ${fields.map(_.name).mkString(", ")}")
- } else if (fields.indexWhere(checkField, ordinal + 1) != -1) {
- throw new AnalysisException(
- s"Ambiguous reference to fields ${fields.filter(checkField).mkString(", ")}")
- } else {
- ordinal
- }
- }
- expr.dataType match {
- case StructType(fields) =>
- val ordinal = findField(fields)
- StructGetField(expr, fields(ordinal), ordinal)
- case ArrayType(StructType(fields), containsNull) =>
- val ordinal = findField(fields)
- ArrayGetField(expr, fields(ordinal), ordinal, containsNull)
- case otherType =>
- throw new AnalysisException(s"GetField is not valid on fields of type $otherType")
- }
- }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org