You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/06/23 21:48:38 UTC
spark git commit: [SPARK-21164][SQL] Remove isTableSample from Sample
and isGenerated from Alias and AttributeReference
Repository: spark
Updated Branches:
refs/heads/master 13c2a4f2f -> 03eb6117a
[SPARK-21164][SQL] Remove isTableSample from Sample and isGenerated from Alias and AttributeReference
## What changes were proposed in this pull request?
`isTableSample` and `isGenerated ` were introduced for SQL Generation respectively by https://github.com/apache/spark/pull/11148 and https://github.com/apache/spark/pull/11050
Since SQL Generation is removed, we do not need to keep `isTableSample`.
## How was this patch tested?
The existing test cases
Author: Xiao Li <ga...@gmail.com>
Closes #18379 from gatorsmile/CleanSample.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/03eb6117
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/03eb6117
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/03eb6117
Branch: refs/heads/master
Commit: 03eb6117affcca21798be25706a39e0d5a2f7288
Parents: 13c2a4f
Author: Xiao Li <ga...@gmail.com>
Authored: Fri Jun 23 14:48:33 2017 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Fri Jun 23 14:48:33 2017 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 8 ++---
.../catalyst/expressions/namedExpressions.scala | 34 +++++++-------------
.../optimizer/RewriteDistinctAggregates.scala | 2 +-
.../spark/sql/catalyst/parser/AstBuilder.scala | 2 +-
.../spark/sql/catalyst/planning/patterns.scala | 4 +--
.../spark/sql/catalyst/plans/QueryPlan.scala | 2 +-
.../catalyst/plans/logical/LogicalPlan.scala | 2 +-
.../plans/logical/basicLogicalOperators.scala | 6 +---
.../catalyst/analysis/AnalysisErrorSuite.scala | 2 +-
.../analysis/UnsupportedOperationsSuite.scala | 2 +-
.../catalyst/optimizer/ColumnPruningSuite.scala | 8 ++---
.../sql/catalyst/parser/PlanParserSuite.scala | 4 +--
.../spark/sql/catalyst/plans/PlanTest.scala | 10 +++---
.../BasicStatsEstimationSuite.scala | 4 +--
.../scala/org/apache/spark/sql/Dataset.scala | 4 +--
15 files changed, 40 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 193082e..7e5ebfc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -874,7 +874,7 @@ class Analyzer(
def newAliases(expressions: Seq[NamedExpression]): Seq[NamedExpression] = {
expressions.map {
- case a: Alias => Alias(a.child, a.name)(isGenerated = a.isGenerated)
+ case a: Alias => Alias(a.child, a.name)()
case other => other
}
}
@@ -1368,7 +1368,7 @@ class Analyzer(
val aggregatedCondition =
Aggregate(
grouping,
- Alias(havingCondition, "havingCondition")(isGenerated = true) :: Nil,
+ Alias(havingCondition, "havingCondition")() :: Nil,
child)
val resolvedOperator = execute(aggregatedCondition)
def resolvedAggregateFilter =
@@ -1424,7 +1424,7 @@ class Analyzer(
try {
val unresolvedSortOrders = sortOrder.filter(s => !s.resolved || containsAggregate(s))
val aliasedOrdering =
- unresolvedSortOrders.map(o => Alias(o.child, "aggOrder")(isGenerated = true))
+ unresolvedSortOrders.map(o => Alias(o.child, "aggOrder")())
val aggregatedOrdering = aggregate.copy(aggregateExpressions = aliasedOrdering)
val resolvedAggregate: Aggregate = execute(aggregatedOrdering).asInstanceOf[Aggregate]
val resolvedAliasedOrdering: Seq[Alias] =
@@ -1935,7 +1935,7 @@ class Analyzer(
leafNondeterministic.distinct.map { e =>
val ne = e match {
case n: NamedExpression => n
- case _ => Alias(e, "_nondeterministic")(isGenerated = true)
+ case _ => Alias(e, "_nondeterministic")()
}
e -> ne
}
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index c842f85..29c3380 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -81,9 +81,6 @@ trait NamedExpression extends Expression {
/** Returns the metadata when an expression is a reference to another expression with metadata. */
def metadata: Metadata = Metadata.empty
- /** Returns true if the expression is generated by Catalyst */
- def isGenerated: java.lang.Boolean = false
-
/** Returns a copy of this expression with a new `exprId`. */
def newInstance(): NamedExpression
@@ -128,13 +125,11 @@ abstract class Attribute extends LeafExpression with NamedExpression with NullIn
* qualified way. Consider the examples tableName.name, subQueryAlias.name.
* tableName and subQueryAlias are possible qualifiers.
* @param explicitMetadata Explicit metadata associated with this alias that overwrites child's.
- * @param isGenerated A flag to indicate if this alias is generated by Catalyst
*/
case class Alias(child: Expression, name: String)(
val exprId: ExprId = NamedExpression.newExprId,
val qualifier: Option[String] = None,
- val explicitMetadata: Option[Metadata] = None,
- override val isGenerated: java.lang.Boolean = false)
+ val explicitMetadata: Option[Metadata] = None)
extends UnaryExpression with NamedExpression {
// Alias(Generator, xx) need to be transformed into Generate(generator, ...)
@@ -159,13 +154,11 @@ case class Alias(child: Expression, name: String)(
}
def newInstance(): NamedExpression =
- Alias(child, name)(
- qualifier = qualifier, explicitMetadata = explicitMetadata, isGenerated = isGenerated)
+ Alias(child, name)(qualifier = qualifier, explicitMetadata = explicitMetadata)
override def toAttribute: Attribute = {
if (resolved) {
- AttributeReference(name, child.dataType, child.nullable, metadata)(
- exprId, qualifier, isGenerated)
+ AttributeReference(name, child.dataType, child.nullable, metadata)(exprId, qualifier)
} else {
UnresolvedAttribute(name)
}
@@ -174,7 +167,7 @@ case class Alias(child: Expression, name: String)(
override def toString: String = s"$child AS $name#${exprId.id}$typeSuffix"
override protected final def otherCopyArgs: Seq[AnyRef] = {
- exprId :: qualifier :: explicitMetadata :: isGenerated :: Nil
+ exprId :: qualifier :: explicitMetadata :: Nil
}
override def hashCode(): Int = {
@@ -207,7 +200,6 @@ case class Alias(child: Expression, name: String)(
* @param qualifier An optional string that can be used to referred to this attribute in a fully
* qualified way. Consider the examples tableName.name, subQueryAlias.name.
* tableName and subQueryAlias are possible qualifiers.
- * @param isGenerated A flag to indicate if this reference is generated by Catalyst
*/
case class AttributeReference(
name: String,
@@ -215,8 +207,7 @@ case class AttributeReference(
nullable: Boolean = true,
override val metadata: Metadata = Metadata.empty)(
val exprId: ExprId = NamedExpression.newExprId,
- val qualifier: Option[String] = None,
- override val isGenerated: java.lang.Boolean = false)
+ val qualifier: Option[String] = None)
extends Attribute with Unevaluable {
/**
@@ -253,8 +244,7 @@ case class AttributeReference(
}
override def newInstance(): AttributeReference =
- AttributeReference(name, dataType, nullable, metadata)(
- qualifier = qualifier, isGenerated = isGenerated)
+ AttributeReference(name, dataType, nullable, metadata)(qualifier = qualifier)
/**
* Returns a copy of this [[AttributeReference]] with changed nullability.
@@ -263,7 +253,7 @@ case class AttributeReference(
if (nullable == newNullability) {
this
} else {
- AttributeReference(name, dataType, newNullability, metadata)(exprId, qualifier, isGenerated)
+ AttributeReference(name, dataType, newNullability, metadata)(exprId, qualifier)
}
}
@@ -271,7 +261,7 @@ case class AttributeReference(
if (name == newName) {
this
} else {
- AttributeReference(newName, dataType, nullable, metadata)(exprId, qualifier, isGenerated)
+ AttributeReference(newName, dataType, nullable, metadata)(exprId, qualifier)
}
}
@@ -282,7 +272,7 @@ case class AttributeReference(
if (newQualifier == qualifier) {
this
} else {
- AttributeReference(name, dataType, nullable, metadata)(exprId, newQualifier, isGenerated)
+ AttributeReference(name, dataType, nullable, metadata)(exprId, newQualifier)
}
}
@@ -290,16 +280,16 @@ case class AttributeReference(
if (exprId == newExprId) {
this
} else {
- AttributeReference(name, dataType, nullable, metadata)(newExprId, qualifier, isGenerated)
+ AttributeReference(name, dataType, nullable, metadata)(newExprId, qualifier)
}
}
override def withMetadata(newMetadata: Metadata): Attribute = {
- AttributeReference(name, dataType, nullable, newMetadata)(exprId, qualifier, isGenerated)
+ AttributeReference(name, dataType, nullable, newMetadata)(exprId, qualifier)
}
override protected final def otherCopyArgs: Seq[AnyRef] = {
- exprId :: qualifier :: isGenerated :: Nil
+ exprId :: qualifier :: Nil
}
/** Used to signal the column used to calculate an eventTime watermark (e.g. a#1-T{delayMs}) */
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
index 3b27cd2..4448ace 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
@@ -134,7 +134,7 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
// Aggregation strategy can handle queries with a single distinct group.
if (distinctAggGroups.size > 1) {
// Create the attributes for the grouping id and the group by clause.
- val gid = AttributeReference("gid", IntegerType, nullable = false)(isGenerated = true)
+ val gid = AttributeReference("gid", IntegerType, nullable = false)()
val groupByMap = a.groupingExpressions.collect {
case ne: NamedExpression => ne -> ne.toAttribute
case e => e -> AttributeReference(e.sql, e.dataType, e.nullable)()
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 315c672..ef79cbc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -627,7 +627,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
validate(fraction >= 0.0 - eps && fraction <= 1.0 + eps,
s"Sampling fraction ($fraction) must be on interval [0, 1]",
ctx)
- Sample(0.0, fraction, withReplacement = false, (math.random * 1000).toInt, query)(true)
+ Sample(0.0, fraction, withReplacement = false, (math.random * 1000).toInt, query)
}
ctx.sampleType.getType match {
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index ef925f9..7f370fb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -80,12 +80,12 @@ object PhysicalOperation extends PredicateHelper {
expr.transform {
case a @ Alias(ref: AttributeReference, name) =>
aliases.get(ref)
- .map(Alias(_, name)(a.exprId, a.qualifier, isGenerated = a.isGenerated))
+ .map(Alias(_, name)(a.exprId, a.qualifier))
.getOrElse(a)
case a: AttributeReference =>
aliases.get(a)
- .map(Alias(_, a.name)(a.exprId, a.qualifier, isGenerated = a.isGenerated)).getOrElse(a)
+ .map(Alias(_, a.name)(a.exprId, a.qualifier)).getOrElse(a)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 1f6d05b..01b3da3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -200,7 +200,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
// normalize that for equality testing, by assigning expr id from 0 incrementally. The
// alias name doesn't matter and should be erased.
val normalizedChild = QueryPlan.normalizeExprId(a.child, allAttributes)
- Alias(normalizedChild, "")(ExprId(id), a.qualifier, isGenerated = a.isGenerated)
+ Alias(normalizedChild, "")(ExprId(id), a.qualifier)
case ar: AttributeReference if allAttributes.indexOf(ar.exprId) == -1 =>
// Top level `AttributeReference` may also be used for output like `Alias`, we should
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index 0c098ac..0d30aa7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -221,7 +221,7 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with QueryPlanConstrai
nameParts: Seq[String],
resolver: Resolver,
attribute: Attribute): Option[(Attribute, List[String])] = {
- if (!attribute.isGenerated && resolver(attribute.name, nameParts.head)) {
+ if (resolver(attribute.name, nameParts.head)) {
Option((attribute.withName(nameParts.head), nameParts.tail.toList))
} else {
None
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index d8f89b1..e89caab 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -807,15 +807,13 @@ case class SubqueryAlias(
* @param withReplacement Whether to sample with replacement.
* @param seed the random seed
* @param child the LogicalPlan
- * @param isTableSample Is created from TABLESAMPLE in the parser.
*/
case class Sample(
lowerBound: Double,
upperBound: Double,
withReplacement: Boolean,
seed: Long,
- child: LogicalPlan)(
- val isTableSample: java.lang.Boolean = false) extends UnaryNode {
+ child: LogicalPlan) extends UnaryNode {
val eps = RandomSampler.roundingEpsilon
val fraction = upperBound - lowerBound
@@ -842,8 +840,6 @@ case class Sample(
// Don't propagate column stats, because we don't know the distribution after a sample operation
Statistics(sizeInBytes, sampledRowCount, hints = childStats.hints)
}
-
- override protected def otherCopyArgs: Seq[AnyRef] = isTableSample :: Nil
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 4ed995e..7311dc3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -573,7 +573,7 @@ class AnalysisErrorSuite extends AnalysisTest {
val plan5 = Filter(
Exists(
Sample(0.0, 0.5, false, 1L,
- Filter(EqualTo(UnresolvedAttribute("a"), b), LocalRelation(b)))().select('b)
+ Filter(EqualTo(UnresolvedAttribute("a"), b), LocalRelation(b))).select('b)
),
LocalRelation(a))
assertAnalysisError(plan5,
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
index c39e372..f68d930 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
@@ -491,7 +491,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
// Other unary operations
testUnaryOperatorInStreamingPlan(
- "sample", Sample(0.1, 1, true, 1L, _)(), expectedMsg = "sampling")
+ "sample", Sample(0.1, 1, true, 1L, _), expectedMsg = "sampling")
testUnaryOperatorInStreamingPlan(
"window", Window(Nil, Nil, Nil, _), expectedMsg = "non-time-based windows")
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
index 0b419e9..08e58d4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ColumnPruningSuite.scala
@@ -349,14 +349,14 @@ class ColumnPruningSuite extends PlanTest {
val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
val x = testRelation.subquery('x)
- val query1 = Sample(0.0, 0.6, false, 11L, x)().select('a)
+ val query1 = Sample(0.0, 0.6, false, 11L, x).select('a)
val optimized1 = Optimize.execute(query1.analyze)
- val expected1 = Sample(0.0, 0.6, false, 11L, x.select('a))()
+ val expected1 = Sample(0.0, 0.6, false, 11L, x.select('a))
comparePlans(optimized1, expected1.analyze)
- val query2 = Sample(0.0, 0.6, false, 11L, x)().select('a as 'aa)
+ val query2 = Sample(0.0, 0.6, false, 11L, x).select('a as 'aa)
val optimized2 = Optimize.execute(query2.analyze)
- val expected2 = Sample(0.0, 0.6, false, 11L, x.select('a))().select('a as 'aa)
+ val expected2 = Sample(0.0, 0.6, false, 11L, x.select('a)).select('a as 'aa)
comparePlans(optimized2, expected2.analyze)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index 0a4ae09..bf15b85 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -411,9 +411,9 @@ class PlanParserSuite extends AnalysisTest {
assertEqual(s"$sql tablesample(100 rows)",
table("t").limit(100).select(star()))
assertEqual(s"$sql tablesample(43 percent) as x",
- Sample(0, .43d, withReplacement = false, 10L, table("t").as("x"))(true).select(star()))
+ Sample(0, .43d, withReplacement = false, 10L, table("t").as("x")).select(star()))
assertEqual(s"$sql tablesample(bucket 4 out of 10) as x",
- Sample(0, .4d, withReplacement = false, 10L, table("t").as("x"))(true).select(star()))
+ Sample(0, .4d, withReplacement = false, 10L, table("t").as("x")).select(star()))
intercept(s"$sql tablesample(bucket 4 out of 10 on x) as x",
"TABLESAMPLE(BUCKET x OUT OF y ON colname) is not supported")
intercept(s"$sql tablesample(bucket 11 out of 10) as x",
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
index 25313af..6883d23 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
@@ -63,14 +63,14 @@ abstract class PlanTest extends SparkFunSuite with PredicateHelper {
*/
protected def normalizePlan(plan: LogicalPlan): LogicalPlan = {
plan transform {
- case filter @ Filter(condition: Expression, child: LogicalPlan) =>
- Filter(splitConjunctivePredicates(condition).map(rewriteEqual(_)).sortBy(_.hashCode())
+ case Filter(condition: Expression, child: LogicalPlan) =>
+ Filter(splitConjunctivePredicates(condition).map(rewriteEqual).sortBy(_.hashCode())
.reduce(And), child)
case sample: Sample =>
- sample.copy(seed = 0L)(true)
- case join @ Join(left, right, joinType, condition) if condition.isDefined =>
+ sample.copy(seed = 0L)
+ case Join(left, right, joinType, condition) if condition.isDefined =>
val newCondition =
- splitConjunctivePredicates(condition.get).map(rewriteEqual(_)).sortBy(_.hashCode())
+ splitConjunctivePredicates(condition.get).map(rewriteEqual).sortBy(_.hashCode())
.reduce(And)
Join(left, right, joinType, Some(newCondition))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala
index e9ed36f..912c5fe 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala
@@ -78,14 +78,14 @@ class BasicStatsEstimationSuite extends StatsEstimationTestBase {
}
test("sample estimation") {
- val sample = Sample(0.0, 0.5, withReplacement = false, (math.random * 1000).toLong, plan)()
+ val sample = Sample(0.0, 0.5, withReplacement = false, (math.random * 1000).toLong, plan)
checkStats(sample, Statistics(sizeInBytes = 60, rowCount = Some(5)))
// Child doesn't have rowCount in stats
val childStats = Statistics(sizeInBytes = 120)
val childPlan = DummyLogicalPlan(childStats, childStats)
val sample2 =
- Sample(0.0, 0.11, withReplacement = false, (math.random * 1000).toLong, childPlan)()
+ Sample(0.0, 0.11, withReplacement = false, (math.random * 1000).toLong, childPlan)
checkStats(sample2, Statistics(sizeInBytes = 14))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/03eb6117/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 767dad3..6e66e92 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -1807,7 +1807,7 @@ class Dataset[T] private[sql](
*/
def sample(withReplacement: Boolean, fraction: Double, seed: Long): Dataset[T] = {
withTypedPlan {
- Sample(0.0, fraction, withReplacement, seed, logicalPlan)()
+ Sample(0.0, fraction, withReplacement, seed, logicalPlan)
}
}
@@ -1863,7 +1863,7 @@ class Dataset[T] private[sql](
val normalizedCumWeights = weights.map(_ / sum).scanLeft(0.0d)(_ + _)
normalizedCumWeights.sliding(2).map { x =>
new Dataset[T](
- sparkSession, Sample(x(0), x(1), withReplacement = false, seed, plan)(), encoder)
+ sparkSession, Sample(x(0), x(1), withReplacement = false, seed, plan), encoder)
}.toArray
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org