You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/11/06 23:47:20 UTC
spark git commit: [SPARK-22330][CORE] Linear containsKey operation
for serialized maps
Repository: spark
Updated Branches:
refs/heads/master 5014d6e25 -> 14a32a647
[SPARK-22330][CORE] Linear containsKey operation for serialized maps
…alization.
## What changes were proposed in this pull request?
Use non-linear containsKey operation for serialized maps, lookup into underlying map.
## How was this patch tested?
unit tests
Author: Alexander Istomin <is...@rutarget.ru>
Closes #19553 from Whoosh/SPARK-22330.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/14a32a64
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/14a32a64
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/14a32a64
Branch: refs/heads/master
Commit: 14a32a647a019ed45793784069fbf077b9c45e60
Parents: 5014d6e
Author: Alexander Istomin <is...@rutarget.ru>
Authored: Tue Nov 7 00:47:16 2017 +0100
Committer: Wenchen Fan <we...@databricks.com>
Committed: Tue Nov 7 00:47:16 2017 +0100
----------------------------------------------------------------------
.../org/apache/spark/api/java/JavaUtils.scala | 9 +++-
.../apache/spark/api/java/JavaUtilsSuite.scala | 49 ++++++++++++++++++++
2 files changed, 57 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/14a32a64/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala
index d650623..fd96052 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala
@@ -43,10 +43,17 @@ private[spark] object JavaUtils {
override def size: Int = underlying.size
+ // Delegate to implementation because AbstractMap implementation iterates over whole key set
+ override def containsKey(key: AnyRef): Boolean = try {
+ underlying.contains(key.asInstanceOf[A])
+ } catch {
+ case _: ClassCastException => false
+ }
+
override def get(key: AnyRef): B = try {
underlying.getOrElse(key.asInstanceOf[A], null.asInstanceOf[B])
} catch {
- case ex: ClassCastException => null.asInstanceOf[B]
+ case _: ClassCastException => null.asInstanceOf[B]
}
override def entrySet: ju.Set[ju.Map.Entry[A, B]] = new ju.AbstractSet[ju.Map.Entry[A, B]] {
http://git-wip-us.apache.org/repos/asf/spark/blob/14a32a64/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala b/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala
new file mode 100644
index 0000000..8e6e3e0
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/api/java/JavaUtilsSuite.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.api.java
+
+import java.io.Serializable
+
+import org.mockito.Mockito._
+
+import org.apache.spark.SparkFunSuite
+
+
+class JavaUtilsSuite extends SparkFunSuite {
+
+ test("containsKey implementation without iteratively entrySet call") {
+ val src = new scala.collection.mutable.HashMap[Double, String]
+ val key: Double = 42.5
+ val key2 = "key"
+
+ src.put(key, "42")
+
+ val map: java.util.Map[Double, String] = spy(JavaUtils.mapAsSerializableJavaMap(src))
+
+ assert(map.containsKey(key))
+
+ // ClassCast checking, shouldn't throw exception
+ assert(!map.containsKey(key2))
+ assert(map.get(key2) == null)
+
+ assert(map.get(key).eq("42"))
+ assert(map.isInstanceOf[Serializable])
+
+ verify(map, never()).entrySet()
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org