You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/04/08 22:57:21 UTC
spark git commit: [SQL][minor] remove duplicated resolveGetField and
update comment
Repository: spark
Updated Branches:
refs/heads/master 55a92ef34 -> 941828054
[SQL][minor] remove duplicated resolveGetField and update comment
It's after https://github.com/apache/spark/pull/5189
Author: Wenchen Fan <cl...@outlook.com>
Closes #5304 from cloud-fan/tmp and squashes the following commits:
c58c9b3 [Wenchen Fan] remove duplicated code and update comment
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/94182805
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/94182805
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/94182805
Branch: refs/heads/master
Commit: 9418280547f962eaf309bfff9986cdd848409643
Parents: 55a92ef
Author: Wenchen Fan <cl...@outlook.com>
Authored: Wed Apr 8 13:57:01 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Apr 8 13:57:01 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 32 +-------------------
.../catalyst/plans/logical/LogicalPlan.scala | 13 +++-----
2 files changed, 6 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/94182805/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 119cb9c..b3aba4f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -293,7 +293,7 @@ class Analyzer(
logDebug(s"Resolving $u to $result")
result
case UnresolvedGetField(child, fieldName) if child.resolved =>
- resolveGetField(child, fieldName)
+ q.resolveGetField(child, fieldName, resolver)
}
}
@@ -313,36 +313,6 @@ class Analyzer(
*/
protected def containsStar(exprs: Seq[Expression]): Boolean =
exprs.exists(_.collect { case _: Star => true }.nonEmpty)
-
- /**
- * Returns the resolved `GetField`, and report error if no desired field or over one
- * desired fields are found.
- */
- protected def resolveGetField(expr: Expression, fieldName: String): Expression = {
- def findField(fields: Array[StructField]): Int = {
- val checkField = (f: StructField) => resolver(f.name, fieldName)
- val ordinal = fields.indexWhere(checkField)
- if (ordinal == -1) {
- throw new AnalysisException(
- s"No such struct field $fieldName in ${fields.map(_.name).mkString(", ")}")
- } else if (fields.indexWhere(checkField, ordinal + 1) != -1) {
- throw new AnalysisException(
- s"Ambiguous reference to fields ${fields.filter(checkField).mkString(", ")}")
- } else {
- ordinal
- }
- }
- expr.dataType match {
- case StructType(fields) =>
- val ordinal = findField(fields)
- StructGetField(expr, fields(ordinal), ordinal)
- case ArrayType(StructType(fields), containsNull) =>
- val ordinal = findField(fields)
- ArrayGetField(expr, fields(ordinal), ordinal, containsNull)
- case otherType =>
- throw new AnalysisException(s"GetField is not valid on fields of type $otherType")
- }
- }
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/94182805/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index 2e9f3aa..d8f5858 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -205,11 +205,10 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
// One match, but we also need to extract the requested nested field.
case Seq((a, nestedFields)) =>
try {
-
- // The foldLeft adds UnresolvedGetField for every remaining parts of the name,
- // and aliased it with the last part of the name.
- // For example, consider name "a.b.c", where "a" is resolved to an existing attribute.
- // Then this will add UnresolvedGetField("b") and UnresolvedGetField("c"), and alias
+ // The foldLeft adds GetFields for every remaining parts of the identifier,
+ // and aliases it with the last part of the identifier.
+ // For example, consider "a.b.c", where "a" is resolved to an existing attribute.
+ // Then this will add GetField("c", GetField("b", a)), and alias
// the final expression as "c".
val fieldExprs = nestedFields.foldLeft(a: Expression)(resolveGetField(_, _, resolver))
val aliasName = nestedFields.last
@@ -234,10 +233,8 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
/**
* Returns the resolved `GetField`, and report error if no desired field or over one
* desired fields are found.
- *
- * TODO: this code is duplicated from Analyzer and should be refactored to avoid this.
*/
- protected def resolveGetField(
+ def resolveGetField(
expr: Expression,
fieldName: String,
resolver: Resolver): Expression = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org