You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2016/03/21 19:00:13 UTC
spark git commit: [SPARK-14004][SQL] NamedExpressions should have at
most one qualifier
Repository: spark
Updated Branches:
refs/heads/master 43ebf7a9c -> 5d8de16e7
[SPARK-14004][SQL] NamedExpressions should have at most one qualifier
## What changes were proposed in this pull request?
This is a more aggressive version of PR #11820, which not only fixes the original problem, but also does the following updates to enforce the at-most-one-qualifier constraint:
- Renames `NamedExpression.qualifiers` to `NamedExpression.qualifier`
- Uses `Option[String]` rather than `Seq[String]` for `NamedExpression.qualifier`
Quoted PR description of #11820 here:
> Current implementations of `AttributeReference.sql` and `Alias.sql` joins all available qualifiers, which is logically wrong. But this implementation mistake doesn't cause any real SQL generation bugs though, since there is always at most one qualifier for any given `AttributeReference` or `Alias`.
## How was this patch tested?
Existing tests should be enough.
Author: Cheng Lian <li...@databricks.com>
Closes #11822 from liancheng/spark-14004-aggressive.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5d8de16e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5d8de16e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5d8de16e
Branch: refs/heads/master
Commit: 5d8de16e715497e9d3b5306abc5bbc45402e9b43
Parents: 43ebf7a
Author: Cheng Lian <li...@databricks.com>
Authored: Mon Mar 21 11:00:09 2016 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Mon Mar 21 11:00:09 2016 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 5 +-
.../spark/sql/catalyst/analysis/Catalog.scala | 10 ++--
.../sql/catalyst/analysis/unresolved.scala | 12 ++--
.../sql/catalyst/catalog/SessionCatalog.scala | 4 +-
.../catalyst/expressions/namedExpressions.scala | 62 ++++++++++----------
.../spark/sql/catalyst/planning/patterns.scala | 4 +-
.../spark/sql/catalyst/plans/QueryPlan.scala | 2 +-
.../catalyst/plans/logical/LogicalPlan.scala | 2 +-
.../catalyst/plans/logical/basicOperators.scala | 4 +-
.../SubexpressionEliminationSuite.scala | 2 +-
.../spark/sql/hive/HiveMetastoreCatalog.scala | 6 +-
.../org/apache/spark/sql/hive/SQLBuilder.scala | 12 ++--
12 files changed, 62 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index ebb3a93..7a08c7d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -446,7 +446,7 @@ class Analyzer(
} transformUp {
case other => other transformExpressions {
case a: Attribute =>
- attributeRewrites.get(a).getOrElse(a).withQualifiers(a.qualifiers)
+ attributeRewrites.get(a).getOrElse(a).withQualifier(a.qualifier)
}
}
newRight
@@ -1467,8 +1467,7 @@ object CleanupAliases extends Rule[LogicalPlan] {
def trimNonTopLevelAliases(e: Expression): Expression = e match {
case a: Alias =>
- Alias(trimAliases(a.child), a.name)(
- a.exprId, a.qualifiers, a.explicitMetadata, a.isGenerated)
+ a.withNewChildren(trimAliases(a.child) :: Nil)
case other => trimAliases(other)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
index 52b284b..2f0a4db 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
@@ -101,13 +101,13 @@ class SimpleCatalog(val conf: CatalystConf) extends Catalog {
if (table == null) {
throw new AnalysisException("Table not found: " + tableName)
}
- val tableWithQualifiers = SubqueryAlias(tableName, table)
+ val qualifiedTable = SubqueryAlias(tableName, table)
// If an alias was specified by the lookup, wrap the plan in a subquery so that attributes are
// properly qualified with this alias.
alias
- .map(a => SubqueryAlias(a, tableWithQualifiers))
- .getOrElse(tableWithQualifiers)
+ .map(a => SubqueryAlias(a, qualifiedTable))
+ .getOrElse(qualifiedTable)
}
override def getTables(databaseName: Option[String]): Seq[(String, Boolean)] = {
@@ -149,11 +149,11 @@ trait OverrideCatalog extends Catalog {
getOverriddenTable(tableIdent) match {
case Some(table) =>
val tableName = getTableName(tableIdent)
- val tableWithQualifiers = SubqueryAlias(tableName, table)
+ val qualifiedTable = SubqueryAlias(tableName, table)
// If an alias was specified by the lookup, wrap the plan in a sub-query so that attributes
// are properly qualified with this alias.
- alias.map(a => SubqueryAlias(a, tableWithQualifiers)).getOrElse(tableWithQualifiers)
+ alias.map(a => SubqueryAlias(a, qualifiedTable)).getOrElse(qualifiedTable)
case None => super.lookupRelation(tableIdent, alias)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 01afa01..9518309 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -59,12 +59,12 @@ case class UnresolvedAttribute(nameParts: Seq[String]) extends Attribute with Un
override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
override def dataType: DataType = throw new UnresolvedException(this, "dataType")
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
- override def qualifiers: Seq[String] = throw new UnresolvedException(this, "qualifiers")
+ override def qualifier: Option[String] = throw new UnresolvedException(this, "qualifier")
override lazy val resolved = false
override def newInstance(): UnresolvedAttribute = this
override def withNullability(newNullability: Boolean): UnresolvedAttribute = this
- override def withQualifiers(newQualifiers: Seq[String]): UnresolvedAttribute = this
+ override def withQualifier(newQualifier: Option[String]): UnresolvedAttribute = this
override def withName(newName: String): UnresolvedAttribute = UnresolvedAttribute.quoted(newName)
override def toString: String = s"'$name"
@@ -158,7 +158,7 @@ abstract class Star extends LeafExpression with NamedExpression {
override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
override def dataType: DataType = throw new UnresolvedException(this, "dataType")
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
- override def qualifiers: Seq[String] = throw new UnresolvedException(this, "qualifiers")
+ override def qualifier: Option[String] = throw new UnresolvedException(this, "qualifier")
override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
override def newInstance(): NamedExpression = throw new UnresolvedException(this, "newInstance")
override lazy val resolved = false
@@ -188,7 +188,7 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu
case None => input.output
// If there is a table, pick out attributes that are part of this table.
case Some(t) => if (t.size == 1) {
- input.output.filter(_.qualifiers.exists(resolver(_, t.head)))
+ input.output.filter(_.qualifier.exists(resolver(_, t.head)))
} else {
List()
}
@@ -243,7 +243,7 @@ case class MultiAlias(child: Expression, names: Seq[String])
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
- override def qualifiers: Seq[String] = throw new UnresolvedException(this, "qualifiers")
+ override def qualifier: Option[String] = throw new UnresolvedException(this, "qualifier")
override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
@@ -298,7 +298,7 @@ case class UnresolvedAlias(child: Expression, aliasName: Option[String] = None)
extends UnaryExpression with NamedExpression with Unevaluable {
override def toAttribute: Attribute = throw new UnresolvedException(this, "toAttribute")
- override def qualifiers: Seq[String] = throw new UnresolvedException(this, "qualifiers")
+ override def qualifier: Option[String] = throw new UnresolvedException(this, "qualifier")
override def exprId: ExprId = throw new UnresolvedException(this, "exprId")
override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
override def dataType: DataType = throw new UnresolvedException(this, "dataType")
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 4dec042..3ac2bcf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -206,10 +206,10 @@ class SessionCatalog(externalCatalog: ExternalCatalog) {
} else {
tempTables.get(name.table)
}
- val tableWithQualifiers = SubqueryAlias(name.table, relation)
+ val qualifiedTable = SubqueryAlias(name.table, relation)
// If an alias was specified by the lookup, wrap the plan in a subquery so that
// attributes are properly qualified with this alias.
- alias.map(a => SubqueryAlias(a, tableWithQualifiers)).getOrElse(tableWithQualifiers)
+ alias.map(a => SubqueryAlias(a, qualifiedTable)).getOrElse(qualifiedTable)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 8c63cce..a5b5758 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -61,10 +61,10 @@ trait NamedExpression extends Expression {
* multiple qualifiers, it is possible that there are other possible way to refer to this
* attribute.
*/
- def qualifiedName: String = (qualifiers.headOption.toSeq :+ name).mkString(".")
+ def qualifiedName: String = (qualifier.toSeq :+ name).mkString(".")
/**
- * All possible qualifiers for the expression.
+ * Optional qualifier for the expression.
*
* For now, since we do not allow using original table name to qualify a column name once the
* table is aliased, this can only be:
@@ -73,7 +73,7 @@ trait NamedExpression extends Expression {
* e.g. top level attributes aliased in the SELECT clause, or column from a LocalRelation.
* 2. Single element: either the table name or the alias name of the table.
*/
- def qualifiers: Seq[String]
+ def qualifier: Option[String]
def toAttribute: Attribute
@@ -102,7 +102,7 @@ abstract class Attribute extends LeafExpression with NamedExpression {
override def references: AttributeSet = AttributeSet(this)
def withNullability(newNullability: Boolean): Attribute
- def withQualifiers(newQualifiers: Seq[String]): Attribute
+ def withQualifier(newQualifier: Option[String]): Attribute
def withName(newName: String): Attribute
override def toAttribute: Attribute = this
@@ -122,7 +122,7 @@ abstract class Attribute extends LeafExpression with NamedExpression {
* @param name The name to be associated with the result of computing [[child]].
* @param exprId A globally unique id used to check if an [[AttributeReference]] refers to this
* alias. Auto-assigned if left blank.
- * @param qualifiers A list of strings that can be used to referred to this attribute in a fully
+ * @param qualifier An optional string that can be used to referred to this attribute in a fully
* qualified way. Consider the examples tableName.name, subQueryAlias.name.
* tableName and subQueryAlias are possible qualifiers.
* @param explicitMetadata Explicit metadata associated with this alias that overwrites child's.
@@ -130,7 +130,7 @@ abstract class Attribute extends LeafExpression with NamedExpression {
*/
case class Alias(child: Expression, name: String)(
val exprId: ExprId = NamedExpression.newExprId,
- val qualifiers: Seq[String] = Nil,
+ val qualifier: Option[String] = None,
val explicitMetadata: Option[Metadata] = None,
override val isGenerated: java.lang.Boolean = false)
extends UnaryExpression with NamedExpression {
@@ -158,12 +158,12 @@ case class Alias(child: Expression, name: String)(
def newInstance(): NamedExpression =
Alias(child, name)(
- qualifiers = qualifiers, explicitMetadata = explicitMetadata, isGenerated = isGenerated)
+ qualifier = qualifier, explicitMetadata = explicitMetadata, isGenerated = isGenerated)
override def toAttribute: Attribute = {
if (resolved) {
AttributeReference(name, child.dataType, child.nullable, metadata)(
- exprId, qualifiers, isGenerated)
+ exprId, qualifier, isGenerated)
} else {
UnresolvedAttribute(name)
}
@@ -172,19 +172,19 @@ case class Alias(child: Expression, name: String)(
override def toString: String = s"$child AS $name#${exprId.id}$typeSuffix"
override protected final def otherCopyArgs: Seq[AnyRef] = {
- exprId :: qualifiers :: explicitMetadata :: isGenerated :: Nil
+ exprId :: qualifier :: explicitMetadata :: isGenerated :: Nil
}
override def equals(other: Any): Boolean = other match {
case a: Alias =>
- name == a.name && exprId == a.exprId && child == a.child && qualifiers == a.qualifiers &&
+ name == a.name && exprId == a.exprId && child == a.child && qualifier == a.qualifier &&
explicitMetadata == a.explicitMetadata
case _ => false
}
override def sql: String = {
- val qualifiersString = if (qualifiers.isEmpty) "" else qualifiers.head + "."
- s"${child.sql} AS $qualifiersString${quoteIdentifier(name)}"
+ val qualifierPrefix = qualifier.map(_ + ".").getOrElse("")
+ s"${child.sql} AS $qualifierPrefix${quoteIdentifier(name)}"
}
}
@@ -197,9 +197,9 @@ case class Alias(child: Expression, name: String)(
* @param metadata The metadata of this attribute.
* @param exprId A globally unique id used to check if different AttributeReferences refer to the
* same attribute.
- * @param qualifiers A list of strings that can be used to referred to this attribute in a fully
- * qualified way. Consider the examples tableName.name, subQueryAlias.name.
- * tableName and subQueryAlias are possible qualifiers.
+ * @param qualifier An optional string that can be used to referred to this attribute in a fully
+ * qualified way. Consider the examples tableName.name, subQueryAlias.name.
+ * tableName and subQueryAlias are possible qualifiers.
* @param isGenerated A flag to indicate if this reference is generated by Catalyst
*/
case class AttributeReference(
@@ -208,7 +208,7 @@ case class AttributeReference(
nullable: Boolean = true,
override val metadata: Metadata = Metadata.empty)(
val exprId: ExprId = NamedExpression.newExprId,
- val qualifiers: Seq[String] = Nil,
+ val qualifier: Option[String] = None,
override val isGenerated: java.lang.Boolean = false)
extends Attribute with Unevaluable {
@@ -220,7 +220,7 @@ case class AttributeReference(
override def equals(other: Any): Boolean = other match {
case ar: AttributeReference =>
name == ar.name && dataType == ar.dataType && nullable == ar.nullable &&
- metadata == ar.metadata && exprId == ar.exprId && qualifiers == ar.qualifiers
+ metadata == ar.metadata && exprId == ar.exprId && qualifier == ar.qualifier
case _ => false
}
@@ -241,13 +241,13 @@ case class AttributeReference(
h = h * 37 + nullable.hashCode()
h = h * 37 + metadata.hashCode()
h = h * 37 + exprId.hashCode()
- h = h * 37 + qualifiers.hashCode()
+ h = h * 37 + qualifier.hashCode()
h
}
override def newInstance(): AttributeReference =
AttributeReference(name, dataType, nullable, metadata)(
- qualifiers = qualifiers, isGenerated = isGenerated)
+ qualifier = qualifier, isGenerated = isGenerated)
/**
* Returns a copy of this [[AttributeReference]] with changed nullability.
@@ -256,7 +256,7 @@ case class AttributeReference(
if (nullable == newNullability) {
this
} else {
- AttributeReference(name, dataType, newNullability, metadata)(exprId, qualifiers, isGenerated)
+ AttributeReference(name, dataType, newNullability, metadata)(exprId, qualifier, isGenerated)
}
}
@@ -264,18 +264,18 @@ case class AttributeReference(
if (name == newName) {
this
} else {
- AttributeReference(newName, dataType, nullable, metadata)(exprId, qualifiers, isGenerated)
+ AttributeReference(newName, dataType, nullable, metadata)(exprId, qualifier, isGenerated)
}
}
/**
- * Returns a copy of this [[AttributeReference]] with new qualifiers.
+ * Returns a copy of this [[AttributeReference]] with new qualifier.
*/
- override def withQualifiers(newQualifiers: Seq[String]): AttributeReference = {
- if (newQualifiers.toSet == qualifiers.toSet) {
+ override def withQualifier(newQualifier: Option[String]): AttributeReference = {
+ if (newQualifier == qualifier) {
this
} else {
- AttributeReference(name, dataType, nullable, metadata)(exprId, newQualifiers, isGenerated)
+ AttributeReference(name, dataType, nullable, metadata)(exprId, newQualifier, isGenerated)
}
}
@@ -283,12 +283,12 @@ case class AttributeReference(
if (exprId == newExprId) {
this
} else {
- AttributeReference(name, dataType, nullable, metadata)(newExprId, qualifiers, isGenerated)
+ AttributeReference(name, dataType, nullable, metadata)(newExprId, qualifier, isGenerated)
}
}
override protected final def otherCopyArgs: Seq[AnyRef] = {
- exprId :: qualifiers :: isGenerated :: Nil
+ exprId :: qualifier :: isGenerated :: Nil
}
override def toString: String = s"$name#${exprId.id}$typeSuffix"
@@ -298,8 +298,8 @@ case class AttributeReference(
override def simpleString: String = s"$name#${exprId.id}: ${dataType.simpleString}"
override def sql: String = {
- val qualifiersString = if (qualifiers.isEmpty) "" else qualifiers.head + "."
- s"$qualifiersString${quoteIdentifier(name)}"
+ val qualifierPrefix = qualifier.map(_ + ".").getOrElse("")
+ s"$qualifierPrefix${quoteIdentifier(name)}"
}
}
@@ -324,10 +324,10 @@ case class PrettyAttribute(
override def withNullability(newNullability: Boolean): Attribute =
throw new UnsupportedOperationException
override def newInstance(): Attribute = throw new UnsupportedOperationException
- override def withQualifiers(newQualifiers: Seq[String]): Attribute =
+ override def withQualifier(newQualifier: Option[String]): Attribute =
throw new UnsupportedOperationException
override def withName(newName: String): Attribute = throw new UnsupportedOperationException
- override def qualifiers: Seq[String] = throw new UnsupportedOperationException
+ override def qualifier: Option[String] = throw new UnsupportedOperationException
override def exprId: ExprId = throw new UnsupportedOperationException
override def nullable: Boolean = throw new UnsupportedOperationException
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index ef74504..ada8424 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -80,12 +80,12 @@ object PhysicalOperation extends PredicateHelper {
expr.transform {
case a @ Alias(ref: AttributeReference, name) =>
aliases.get(ref)
- .map(Alias(_, name)(a.exprId, a.qualifiers, isGenerated = a.isGenerated))
+ .map(Alias(_, name)(a.exprId, a.qualifier, isGenerated = a.isGenerated))
.getOrElse(a)
case a: AttributeReference =>
aliases.get(a)
- .map(Alias(_, a.name)(a.exprId, a.qualifiers, isGenerated = a.isGenerated)).getOrElse(a)
+ .map(Alias(_, a.name)(a.exprId, a.qualifier, isGenerated = a.isGenerated)).getOrElse(a)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 920e989..e9bfa09 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -298,7 +298,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
// As the root of the expression, Alias will always take an arbitrary exprId, we need
// to erase that for equality testing.
val cleanedExprId =
- Alias(a.child, a.name)(ExprId(-1), a.qualifiers, isGenerated = a.isGenerated)
+ Alias(a.child, a.name)(ExprId(-1), a.qualifier, isGenerated = a.isGenerated)
BindReferences.bindReference(cleanedExprId, allAttributes, allowFailures = true)
case other =>
BindReferences.bindReference(other, allAttributes, allowFailures = true)
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index 0e02ad6..01c1fa4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -177,7 +177,7 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
resolver: Resolver,
attribute: Attribute): Option[(Attribute, List[String])] = {
assert(nameParts.length > 1)
- if (attribute.qualifiers.exists(resolver(_, nameParts.head))) {
+ if (attribute.qualifier.exists(resolver(_, nameParts.head))) {
// At least one qualifier matches. See if remaining parts match.
val remainingParts = nameParts.tail
resolveAsColumn(remainingParts, resolver, attribute)
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
index ccc9916..09c200f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
@@ -94,7 +94,7 @@ case class Generate(
def output: Seq[Attribute] = {
val qualified = qualifier.map(q =>
// prepend the new qualifier to the existed one
- generatorOutput.map(a => a.withQualifiers(q +: a.qualifiers))
+ generatorOutput.map(a => a.withQualifier(Some(q)))
).getOrElse(generatorOutput)
if (join) child.output ++ qualified else qualified
@@ -615,7 +615,7 @@ case class LocalLimit(limitExpr: Expression, child: LogicalPlan) extends UnaryNo
case class SubqueryAlias(alias: String, child: LogicalPlan) extends UnaryNode {
- override def output: Seq[Attribute] = child.output.map(_.withQualifiers(alias :: Nil))
+ override def output: Seq[Attribute] = child.output.map(_.withQualifier(Some(alias)))
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala
index 5d688e2..90e97d7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SubexpressionEliminationSuite.scala
@@ -25,7 +25,7 @@ class SubexpressionEliminationSuite extends SparkFunSuite {
val a: AttributeReference = AttributeReference("name", IntegerType)()
val b1 = a.withName("name2").withExprId(id)
val b2 = a.withExprId(id)
- val b3 = a.withQualifiers("qualifierName" :: Nil)
+ val b3 = a.withQualifier(Some("qualifierName"))
assert(b1 != b2)
assert(a != b1)
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 69bccfb..27e4cfc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -421,10 +421,10 @@ private[hive] class HiveMetastoreCatalog(val client: HiveClient, hive: HiveConte
if (table.properties.get("spark.sql.sources.provider").isDefined) {
val dataSourceTable = cachedDataSourceTables(qualifiedTableName)
- val tableWithQualifiers = SubqueryAlias(qualifiedTableName.name, dataSourceTable)
+ val qualifiedTable = SubqueryAlias(qualifiedTableName.name, dataSourceTable)
// Then, if alias is specified, wrap the table with a Subquery using the alias.
// Otherwise, wrap the table with a Subquery using the table name.
- alias.map(a => SubqueryAlias(a, tableWithQualifiers)).getOrElse(tableWithQualifiers)
+ alias.map(a => SubqueryAlias(a, qualifiedTable)).getOrElse(qualifiedTable)
} else if (table.tableType == CatalogTableType.VIRTUAL_VIEW) {
val viewText = table.viewText.getOrElse(sys.error("Invalid view without text."))
alias match {
@@ -935,7 +935,7 @@ private[hive] case class MetastoreRelation(
HiveMetastoreTypes.toDataType(f.dataType),
// Since data can be dumped in randomly with no validation, everything is nullable.
nullable = true
- )(qualifiers = Seq(alias.getOrElse(tableName)))
+ )(qualifier = Some(alias.getOrElse(tableName)))
}
/** PartitionKey attributes */
http://git-wip-us.apache.org/repos/asf/spark/blob/5d8de16e/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
index b2196c4..e54358e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
@@ -50,7 +50,7 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
def toSQL: String = {
val canonicalizedPlan = Canonicalizer.execute(logicalPlan)
val outputNames = logicalPlan.output.map(_.name)
- val qualifiers = logicalPlan.output.flatMap(_.qualifiers).distinct
+ val qualifiers = logicalPlan.output.flatMap(_.qualifier).distinct
// Keep the qualifier information by using it as sub-query name, if there is only one qualifier
// present.
@@ -63,7 +63,7 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
// Canonicalizer will remove all naming information, we should add it back by adding an extra
// Project and alias the outputs.
val aliasedOutput = canonicalizedPlan.output.zip(outputNames).map {
- case (attr, name) => Alias(attr.withQualifiers(Nil), name)()
+ case (attr, name) => Alias(attr.withQualifier(None), name)()
}
val finalPlan = Project(aliasedOutput, SubqueryAlias(finalName, canonicalizedPlan))
@@ -411,9 +411,9 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
object NormalizedAttribute extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressions {
case a: AttributeReference =>
- AttributeReference(normalizedName(a), a.dataType)(exprId = a.exprId, qualifiers = Nil)
+ AttributeReference(normalizedName(a), a.dataType)(exprId = a.exprId, qualifier = None)
case a: Alias =>
- Alias(a.child, normalizedName(a))(exprId = a.exprId, qualifiers = Nil)
+ Alias(a.child, normalizedName(a))(exprId = a.exprId, qualifier = None)
}
}
@@ -513,10 +513,10 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
object ExtractSQLTable {
def unapply(plan: LogicalPlan): Option[SQLTable] = plan match {
case l @ LogicalRelation(_, _, Some(TableIdentifier(table, Some(database)))) =>
- Some(SQLTable(database, table, l.output.map(_.withQualifiers(Nil))))
+ Some(SQLTable(database, table, l.output.map(_.withQualifier(None))))
case m: MetastoreRelation =>
- Some(SQLTable(m.databaseName, m.tableName, m.output.map(_.withQualifiers(Nil))))
+ Some(SQLTable(m.databaseName, m.tableName, m.output.map(_.withQualifier(None))))
case _ => None
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org