You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/04/17 02:32:48 UTC
spark git commit: SPARK-6927 [SQL] Sorting Error when codegen on
Repository: spark
Updated Branches:
refs/heads/master 04e44b37c -> 5fe434335
SPARK-6927 [SQL] Sorting Error when codegen on
Fix this error by adding BinaryType comparor in GenerateOrdering.
JIRA https://issues.apache.org/jira/browse/SPARK-6927
Author: 云峤 <ch...@alibaba-inc.com>
Closes #5524 from kaka1992/fix-codegen-sort and squashes the following commits:
d7e2afe [云峤] fix codegen sorting error
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fe43433
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fe43433
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fe43433
Branch: refs/heads/master
Commit: 5fe43433529346788e8c343d338a5b7dc169cf58
Parents: 04e44b3
Author: 云峤 <ch...@alibaba-inc.com>
Authored: Thu Apr 16 17:32:42 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Thu Apr 16 17:32:42 2015 -0700
----------------------------------------------------------------------
.../expressions/codegen/GenerateOrdering.scala | 14 +++++++++++++-
.../org/apache/spark/sql/SQLQuerySuite.scala | 20 ++++++++++++++++++++
2 files changed, 33 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe43433/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
index 0db29eb..fc2a2b6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.expressions.codegen
import org.apache.spark.Logging
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.types.{StringType, NumericType}
+import org.apache.spark.sql.types.{BinaryType, StringType, NumericType}
/**
* Generates bytecode for an [[Ordering]] of [[Row Rows]] for a given set of
@@ -43,6 +43,18 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[Row]] wit
val evalB = expressionEvaluator(order.child)
val compare = order.child.dataType match {
+ case BinaryType =>
+ q"""
+ val x = ${if (order.direction == Ascending) evalA.primitiveTerm else evalB.primitiveTerm}
+ val y = ${if (order.direction != Ascending) evalB.primitiveTerm else evalA.primitiveTerm}
+ var i = 0
+ while (i < x.length && i < y.length) {
+ val res = x(i).compareTo(y(i))
+ if (res != 0) return res
+ i = i+1
+ }
+ return x.length - y.length
+ """
case _: NumericType =>
q"""
val comp = ${evalA.primitiveTerm} - ${evalB.primitiveTerm}
http://git-wip-us.apache.org/repos/asf/spark/blob/5fe43433/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index d739e55..9e02e69 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -398,6 +398,26 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
setConf(SQLConf.EXTERNAL_SORT, before.toString)
}
+ test("SPARK-6927 sorting with codegen on") {
+ val externalbefore = conf.externalSortEnabled
+ val codegenbefore = conf.codegenEnabled
+ setConf(SQLConf.EXTERNAL_SORT, "false")
+ setConf(SQLConf.CODEGEN_ENABLED, "true")
+ sortTest()
+ setConf(SQLConf.EXTERNAL_SORT, externalbefore.toString)
+ setConf(SQLConf.CODEGEN_ENABLED, codegenbefore.toString)
+ }
+
+ test("SPARK-6927 external sorting with codegen on") {
+ val externalbefore = conf.externalSortEnabled
+ val codegenbefore = conf.codegenEnabled
+ setConf(SQLConf.CODEGEN_ENABLED, "true")
+ setConf(SQLConf.EXTERNAL_SORT, "true")
+ sortTest()
+ setConf(SQLConf.EXTERNAL_SORT, externalbefore.toString)
+ setConf(SQLConf.CODEGEN_ENABLED, codegenbefore.toString)
+ }
+
test("limit") {
checkAnswer(
sql("SELECT * FROM testData LIMIT 10"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org