You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/28 18:43:16 UTC
spark git commit: [SPARK-9373][SQL] follow up for StructType support
in Tungsten projection.
Repository: spark
Updated Branches:
refs/heads/master 5a2330e54 -> c740bed17
[SPARK-9373][SQL] follow up for StructType support in Tungsten projection.
Author: Reynold Xin <rx...@databricks.com>
Closes #7720 from rxin/struct-followup and squashes the following commits:
d9757f5 [Reynold Xin] [SPARK-9373][SQL] follow up for StructType support in Tungsten projection.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c740bed1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c740bed1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c740bed1
Branch: refs/heads/master
Commit: c740bed17215a9608c9eb9d80ffdf0fcf72c3911
Parents: 5a2330e
Author: Reynold Xin <rx...@databricks.com>
Authored: Tue Jul 28 09:43:12 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Jul 28 09:43:12 2015 -0700
----------------------------------------------------------------------
.../catalyst/expressions/UnsafeRowWriters.java | 6 +--
.../codegen/GenerateUnsafeProjection.scala | 40 +++++++++-----------
.../spark/sql/execution/SparkStrategies.scala | 3 +-
3 files changed, 23 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c740bed1/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java
index 8fdd739..32faad3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRowWriters.java
@@ -47,7 +47,7 @@ public class UnsafeRowWriters {
target.getBaseObject(), offset + ((numBytes >> 3) << 3), 0L);
}
- // Write the string to the variable length portion.
+ // Write the bytes to the variable length portion.
input.writeToMemory(target.getBaseObject(), offset);
// Set the fixed length portion.
@@ -73,7 +73,7 @@ public class UnsafeRowWriters {
target.getBaseObject(), offset + ((numBytes >> 3) << 3), 0L);
}
- // Write the string to the variable length portion.
+ // Write the bytes to the variable length portion.
ByteArray.writeToMemory(input, target.getBaseObject(), offset);
// Set the fixed length portion.
@@ -115,7 +115,7 @@ public class UnsafeRowWriters {
target.getBaseObject(), offset + ((numBytes >> 3) << 3), 0L);
}
- // Write the string to the variable length portion.
+ // Write the bytes to the variable length portion.
row.writeToMemory(target.getBaseObject(), offset);
// Set the fixed length portion.
http://git-wip-us.apache.org/repos/asf/spark/blob/c740bed1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
index 3e87f72..9a4c00e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
@@ -62,14 +62,10 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
val cursor = ctx.freshName("cursor")
val numBytes = ctx.freshName("numBytes")
- val exprs = expressions.zipWithIndex.map { case (e, i) =>
- e.dataType match {
- case st: StructType =>
- createCodeForStruct(ctx, e.gen(ctx), st)
- case _ =>
- e.gen(ctx)
- }
- }
+ val exprs = expressions.map { e => e.dataType match {
+ case st: StructType => createCodeForStruct(ctx, e.gen(ctx), st)
+ case _ => e.gen(ctx)
+ }}
val allExprs = exprs.map(_.code).mkString("\n")
val fixedSize = 8 * exprs.length + UnsafeRow.calculateBitSetWidthInBytes(exprs.length)
@@ -153,20 +149,20 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
val exprs: Seq[GeneratedExpressionCode] = schema.map(_.dataType).zipWithIndex.map {
case (dt, i) => dt match {
- case st: StructType =>
- val nestedStructEv = GeneratedExpressionCode(
- code = "",
- isNull = s"${input.primitive}.isNullAt($i)",
- primitive = s"${ctx.getColumn(input.primitive, dt, i)}"
- )
- createCodeForStruct(ctx, nestedStructEv, st)
- case _ =>
- GeneratedExpressionCode(
- code = "",
- isNull = s"${input.primitive}.isNullAt($i)",
- primitive = s"${ctx.getColumn(input.primitive, dt, i)}"
- )
- }
+ case st: StructType =>
+ val nestedStructEv = GeneratedExpressionCode(
+ code = "",
+ isNull = s"${input.primitive}.isNullAt($i)",
+ primitive = s"${ctx.getColumn(input.primitive, dt, i)}"
+ )
+ createCodeForStruct(ctx, nestedStructEv, st)
+ case _ =>
+ GeneratedExpressionCode(
+ code = "",
+ isNull = s"${input.primitive}.isNullAt($i)",
+ primitive = s"${ctx.getColumn(input.primitive, dt, i)}"
+ )
+ }
}
val allExprs = exprs.map(_.code).mkString("\n")
http://git-wip-us.apache.org/repos/asf/spark/blob/c740bed1/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 314b85f..f3ef066 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -339,7 +339,8 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
* if necessary.
*/
def getSortOperator(sortExprs: Seq[SortOrder], global: Boolean, child: SparkPlan): SparkPlan = {
- if (sqlContext.conf.unsafeEnabled && UnsafeExternalSort.supportsSchema(child.schema)) {
+ if (sqlContext.conf.unsafeEnabled && sqlContext.conf.codegenEnabled &&
+ UnsafeExternalSort.supportsSchema(child.schema)) {
execution.UnsafeExternalSort(sortExprs, global, child)
} else if (sqlContext.conf.externalSortEnabled) {
execution.ExternalSort(sortExprs, global, child)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org