You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2016/08/25 09:49:50 UTC
spark git commit: Revert "[SPARK-17061][SPARK-17093][SQL] MapObjects`
should make copies of unsafe-backed data"
Repository: spark
Updated Branches:
refs/heads/branch-2.0 fb1c69714 -> 88481ea21
Revert "[SPARK-17061][SPARK-17093][SQL] MapObjects` should make copies of unsafe-backed data"
This reverts commit fb1c697143a5bb2df69d9f2c9cbddc4eb526f047.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/88481ea2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/88481ea2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/88481ea2
Branch: refs/heads/branch-2.0
Commit: 88481ea2169e0813cfc326eb1440ddaaf3110f4a
Parents: fb1c697
Author: Herman van Hovell <hv...@databricks.com>
Authored: Thu Aug 25 11:48:13 2016 +0200
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Thu Aug 25 11:48:13 2016 +0200
----------------------------------------------------------------------
.../maven_app_core/src/main/java/SimpleApp.java | 41 --------------------
.../catalyst/expressions/objects/objects.scala | 12 +-----
.../expressions/ExpressionEvalHelper.scala | 2 +-
3 files changed, 2 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/88481ea2/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
----------------------------------------------------------------------
diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
index 059e51b..5217689 100644
--- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
+++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
@@ -18,20 +18,11 @@
import org.apache.spark.api.java.*;
import org.apache.spark.api.java.function.Function;
-<<<<<<< HEAD:dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
public class SimpleApp {
public static void main(String[] args) {
String logFile = "input.txt";
JavaSparkContext sc = new JavaSparkContext("local", "Simple App");
JavaRDD<String> logData = sc.textFile(logFile).cache();
-=======
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions.objects.Invoke
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData}
-import org.apache.spark.sql.types.{IntegerType, ObjectType}
->>>>>>> e0b20f9... [SPARK-17061][SPARK-17093][SQL] MapObjects` should make copies of unsafe-backed data:sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
long numAs = logData.filter(new Function<String, Boolean>() {
public Boolean call(String s) { return s.contains("a"); }
@@ -48,36 +39,4 @@ import org.apache.spark.sql.types.{IntegerType, ObjectType}
System.out.println("Test succeeded");
sc.stop();
}
-
- test("MapObjects should make copies of unsafe-backed data") {
- // test UnsafeRow-backed data
- val structEncoder = ExpressionEncoder[Array[Tuple2[java.lang.Integer, java.lang.Integer]]]
- val structInputRow = InternalRow.fromSeq(Seq(Array((1, 2), (3, 4))))
- val structExpected = new GenericArrayData(
- Array(InternalRow.fromSeq(Seq(1, 2)), InternalRow.fromSeq(Seq(3, 4))))
- checkEvalutionWithUnsafeProjection(
- structEncoder.serializer.head, structExpected, structInputRow)
-
- // test UnsafeArray-backed data
- val arrayEncoder = ExpressionEncoder[Array[Array[Int]]]
- val arrayInputRow = InternalRow.fromSeq(Seq(Array(Array(1, 2), Array(3, 4))))
- val arrayExpected = new GenericArrayData(
- Array(new GenericArrayData(Array(1, 2)), new GenericArrayData(Array(3, 4))))
- checkEvalutionWithUnsafeProjection(
- arrayEncoder.serializer.head, arrayExpected, arrayInputRow)
-
- // test UnsafeMap-backed data
- val mapEncoder = ExpressionEncoder[Array[Map[Int, Int]]]
- val mapInputRow = InternalRow.fromSeq(Seq(Array(
- Map(1 -> 100, 2 -> 200), Map(3 -> 300, 4 -> 400))))
- val mapExpected = new GenericArrayData(Seq(
- new ArrayBasedMapData(
- new GenericArrayData(Array(1, 2)),
- new GenericArrayData(Array(100, 200))),
- new ArrayBasedMapData(
- new GenericArrayData(Array(3, 4)),
- new GenericArrayData(Array(300, 400)))))
- checkEvalutionWithUnsafeProjection(
- mapEncoder.serializer.head, mapExpected, mapInputRow)
- }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/88481ea2/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 1cdda53..37ec1a6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -481,16 +481,6 @@ case class MapObjects private(
s"$seq == null ? $array[$loopIndex] : $seq.apply($loopIndex)"
}
- // Make a copy of the data if it's unsafe-backed
- def makeCopyIfInstanceOf(clazz: Class[_ <: Any], value: String) =
- s"$value instanceof ${clazz.getSimpleName}? ${value}.copy() : $value"
- val genFunctionValue = lambdaFunction.dataType match {
- case StructType(_) => makeCopyIfInstanceOf(classOf[UnsafeRow], genFunction.value)
- case ArrayType(_, _) => makeCopyIfInstanceOf(classOf[UnsafeArrayData], genFunction.value)
- case MapType(_, _, _) => makeCopyIfInstanceOf(classOf[UnsafeMapData], genFunction.value)
- case _ => genFunction.value
- }
-
val loopNullCheck = inputDataType match {
case _: ArrayType => s"$loopIsNull = ${genInputData.value}.isNullAt($loopIndex);"
// The element of primitive array will never be null.
@@ -518,7 +508,7 @@ case class MapObjects private(
if (${genFunction.isNull}) {
$convertedArray[$loopIndex] = null;
} else {
- $convertedArray[$loopIndex] = $genFunctionValue;
+ $convertedArray[$loopIndex] = ${genFunction.value};
}
$loopIndex += 1;
http://git-wip-us.apache.org/repos/asf/spark/blob/88481ea2/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
index 668543a..d6a9672 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
@@ -136,7 +136,7 @@ trait ExpressionEvalHelper extends GeneratorDrivenPropertyChecks {
// some expression is reusing variable names across different instances.
// This behavior is tested in ExpressionEvalHelperSuite.
val plan = generateProject(
- UnsafeProjection.create(
+ GenerateUnsafeProjection.generate(
Alias(expression, s"Optimized($expression)1")() ::
Alias(expression, s"Optimized($expression)2")() :: Nil),
expression)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org