You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2019/03/12 16:14:37 UTC

[flink] branch release-1.8 updated (db0b874 -> ce7aea0)

This is an automated email from the ASF dual-hosted git repository.

aljoscha pushed a change to branch release-1.8
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from db0b874  [FLINK-11420][datastream] Fix duplicate and createInstance methods of CoGroupedStreams.UnionSerializer
     new 935ea92  [FLINK-11865] Extract compileCbf from TraversableSerializer
     new ce7aea0  [FLINK-11865] Cache compiled CanBuildFrom values in TraversableSerializer

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 flink-scala/pom.xml                                |  5 ++
 .../scala/typeutils/TraversableSerializer.scala    | 86 +++++++++++++++++++---
 .../scala/runtime/TraversableSerializerTest.scala  | 57 +++++++++++++-
 3 files changed, 133 insertions(+), 15 deletions(-)


[flink] 02/02: [FLINK-11865] Cache compiled CanBuildFrom values in TraversableSerializer

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

aljoscha pushed a commit to branch release-1.8
in repository https://gitbox.apache.org/repos/asf/flink.git

commit ce7aea06785d8e06fbacffe9c6100fe9fbb02f2a
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Mon Mar 11 11:16:32 2019 +0100

    [FLINK-11865] Cache compiled CanBuildFrom values in TraversableSerializer
---
 flink-scala/pom.xml                                |  5 ++
 .../scala/typeutils/TraversableSerializer.scala    | 80 ++++++++++++++++++----
 .../scala/runtime/TraversableSerializerTest.scala  | 57 ++++++++++++++-
 3 files changed, 124 insertions(+), 18 deletions(-)

diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 4a8089c..c3bccb4 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -70,6 +70,11 @@ under the License.
 			<artifactId>scala-compiler</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.flink</groupId>
+			<artifactId>flink-shaded-guava</artifactId>
+		</dependency>
+
 		<!-- test dependencies -->
 
 		<dependency>
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
index dcf5212..cb90249 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
@@ -18,16 +18,19 @@
 package org.apache.flink.api.scala.typeutils
 
 import java.io.ObjectInputStream
+import java.util.concurrent.Callable
 
 import org.apache.flink.annotation.Internal
 import org.apache.flink.api.common.typeutils._
 import org.apache.flink.core.memory.{DataInputView, DataOutputView}
+import org.apache.flink.shaded.guava18.com.google.common.cache.{Cache, CacheBuilder}
 
 import scala.collection.generic.CanBuildFrom
+import scala.ref.WeakReference
 
 /**
- * Serializer for Scala Collections.
- */
+  * Serializer for Scala Collections.
+  */
 @Internal
 @SerialVersionUID(7522917416391312410L)
 class TraversableSerializer[T <: TraversableOnce[E], E](
@@ -47,7 +50,7 @@ class TraversableSerializer[T <: TraversableOnce[E], E](
     val cl = Thread.currentThread().getContextClassLoader
     TraversableSerializer.compileCbf(cl, code)
   }
-  
+
   override def duplicate = {
     val duplicateElementSerializer = elementSerializer.duplicate()
     if (duplicateElementSerializer eq elementSerializer) {
@@ -168,20 +171,67 @@ class TraversableSerializer[T <: TraversableOnce[E], E](
 }
 
 object TraversableSerializer {
-  
-  def compileCbf[T, E](classLoader: ClassLoader, code: String): CanBuildFrom[T, E, T] = {
-    compileCbfInternal(classLoader, code)
+
+  private val CACHE: Cache[Key, CanBuildFrom[_, _, _]] = CacheBuilder.newBuilder()
+    .weakValues()
+    .maximumSize(128)
+    .build()
+
+  def compileCbf[T, E](classLoader: ClassLoader, cbfCode: String): CanBuildFrom[T, E, T] = {
+    val key = Key(classLoader, cbfCode)
+
+    CACHE
+      .get(key, LazyRuntimeCompiler(classLoader, cbfCode))
+      .asInstanceOf[CanBuildFrom[T, E, T]]
   }
 
-  private def compileCbfInternal[T, E](classLoader: ClassLoader, code: String): CanBuildFrom[T, E, T] = {
-    import scala.reflect.runtime.universe._
-    import scala.tools.reflect.ToolBox
+  object Key {
 
-    val tb = runtimeMirror(classLoader).mkToolBox()
-    val tree = tb.parse(code)
-    val compiled = tb.compile(tree)
-    val cbf = compiled()
-    cbf.asInstanceOf[CanBuildFrom[T, E, T]]
+    def apply(classLoader: ClassLoader, cbfCode: String): Key = {
+      val hashCode = System.identityHashCode(classLoader)
+      val weakReference = WeakReference(classLoader)
+      Key(hashCode, weakReference, cbfCode)
+    }
+  }
+
+  case class Key(classLoaderHash: Int,
+                 classLoaderRef: WeakReference[ClassLoader],
+                 cbfCode: String) {
+
+    override def hashCode(): Int = classLoaderHash * 37 + cbfCode.hashCode
+
+    override def equals(obj: Any): Boolean = {
+      obj match {
+        case Key(thatHashCode, thatClassLoaderRef, thatCbfCode) => 
+          (this.classLoaderHash == thatHashCode) && 
+          (this.classLoaderRef.get == thatClassLoaderRef.get) &&
+          (this.cbfCode == thatCbfCode)
+        
+        case _ =>
+          false
+      }
+    }
   }
-  
+
+  private case class LazyRuntimeCompiler[T, E](classLoader: ClassLoader,
+                                               code: String)
+    extends Callable[CanBuildFrom[T, E, T]] {
+
+    override def call(): CanBuildFrom[T, E, T] = compileCbfInternal(classLoader, code)
+
+    private def compileCbfInternal(classLoader: ClassLoader, code: String):
+    CanBuildFrom[T, E, T] = {
+
+      import scala.reflect.runtime.universe._
+      import scala.tools.reflect.ToolBox
+
+      val tb = runtimeMirror(classLoader).mkToolBox()
+      val tree = tb.parse(code)
+      val compiled = tb.compile(tree)
+      val cbf = compiled()
+      cbf.asInstanceOf[CanBuildFrom[T, E, T]]
+    }
+  }
+
 }
+
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala
index 3df28ed..a637f24 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TraversableSerializerTest.scala
@@ -24,10 +24,12 @@ import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer
 import org.apache.flink.api.scala._
 import org.apache.flink.api.scala.typeutils.TraversableSerializer
 import org.junit.Assert._
-import org.junit.{Assert, Ignore, Test}
+import org.junit.{Assert, Test}
 
-import scala.collection.immutable.{BitSet, LinearSeq, SortedSet}
-import scala.collection.{SortedMap, mutable}
+import scala.collection.immutable.{BitSet, LinearSeq}
+import scala.collection.mutable
+import scala.ref.WeakReference
+import scala.reflect.internal.util.ScalaClassLoader.URLClassLoader
 
 class TraversableSerializerTest {
 
@@ -97,6 +99,55 @@ class TraversableSerializerTest {
     runTests(testData)
   }
 
+  @Test
+  def sameClassLoaderAndCodeShouldProvideEqualKeys(): Unit = {
+    val classLoaderA = new URLClassLoader(Seq.empty[java.net.URL], null)
+
+    val keyA = TraversableSerializer.Key(classLoaderA, "code")
+    val keyB = TraversableSerializer.Key(classLoaderA, "code")
+
+    assertEquals(keyA, keyB)
+  }
+
+  @Test
+  def differentClassLoadersProvideNonEqualKeys(): Unit = {
+    val classLoaderA = new URLClassLoader(Seq.empty[java.net.URL], null)
+    val classLoaderB = new URLClassLoader(Seq.empty[java.net.URL], null)
+    
+    val keyA = TraversableSerializer.Key(classLoaderA, "code")
+    val keyB = TraversableSerializer.Key(classLoaderB, "code")
+    
+    assertNotEquals(keyA, keyB)
+  }
+
+  @Test
+  def expiredReferenceShouldProduceNonEqualKeys(): Unit = {
+    val classLoaderA = new URLClassLoader(Seq.empty[java.net.URL], null)
+
+    val keyA = TraversableSerializer.Key(classLoaderA, "code")
+    val keyB = keyA.copy(classLoaderRef = WeakReference(null)) 
+
+    assertNotEquals(keyA, keyB)
+  }
+
+  @Test
+  def bootStrapClassLoaderShouldProduceTheSameKeys(): Unit = {
+    val keyA = TraversableSerializer.Key(null, "a")
+    val keyB = TraversableSerializer.Key(null, "a")
+
+    assertEquals(keyA, keyB)
+  }
+
+  @Test
+  def differentCanBuildFromCodeShouldProduceDifferentKeys(): Unit = {
+    val classLoaderA = new URLClassLoader(Seq.empty[java.net.URL], null)
+
+    val keyA = TraversableSerializer.Key(classLoaderA, "a")
+    val keyB = TraversableSerializer.Key(classLoaderA, "b")
+
+    assertNotEquals(keyA, keyB)
+  }
+  
   private final def runTests[T : TypeInformation](instances: Array[T]) {
     try {
       val typeInfo = implicitly[TypeInformation[T]]


[flink] 01/02: [FLINK-11865] Extract compileCbf from TraversableSerializer

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

aljoscha pushed a commit to branch release-1.8
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 935ea9219fc3618fc3f0346d474fed6709601e88
Author: Igal Shilman <ig...@data-artisans.com>
AuthorDate: Mon Mar 11 09:59:30 2019 +0100

    [FLINK-11865] Extract compileCbf from TraversableSerializer
---
 .../scala/typeutils/TraversableSerializer.scala    | 34 +++++++++++++++-------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
index 82d21ec..dcf5212 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/TraversableSerializer.scala
@@ -17,7 +17,7 @@
  */
 package org.apache.flink.api.scala.typeutils
 
-import java.io.{ObjectInputStream, ObjectOutputStream}
+import java.io.ObjectInputStream
 
 import org.apache.flink.annotation.Internal
 import org.apache.flink.api.common.typeutils._
@@ -44,17 +44,10 @@ class TraversableSerializer[T <: TraversableOnce[E], E](
   protected def legacyCbfCode: String = null
 
   def compileCbf(code: String): CanBuildFrom[T, E, T] = {
-
-    import scala.reflect.runtime.universe._
-    import scala.tools.reflect.ToolBox
-
-    val tb = runtimeMirror(Thread.currentThread().getContextClassLoader).mkToolBox()
-    val tree = tb.parse(code)
-    val compiled = tb.compile(tree)
-    val cbf = compiled()
-    cbf.asInstanceOf[CanBuildFrom[T, E, T]]
+    val cl = Thread.currentThread().getContextClassLoader
+    TraversableSerializer.compileCbf(cl, code)
   }
-
+  
   override def duplicate = {
     val duplicateElementSerializer = elementSerializer.duplicate()
     if (duplicateElementSerializer eq elementSerializer) {
@@ -173,3 +166,22 @@ class TraversableSerializer[T <: TraversableOnce[E], E](
     new TraversableSerializerSnapshot[T, E](this)
   }
 }
+
+object TraversableSerializer {
+  
+  def compileCbf[T, E](classLoader: ClassLoader, code: String): CanBuildFrom[T, E, T] = {
+    compileCbfInternal(classLoader, code)
+  }
+
+  private def compileCbfInternal[T, E](classLoader: ClassLoader, code: String): CanBuildFrom[T, E, T] = {
+    import scala.reflect.runtime.universe._
+    import scala.tools.reflect.ToolBox
+
+    val tb = runtimeMirror(classLoader).mkToolBox()
+    val tree = tb.parse(code)
+    val compiled = tb.compile(tree)
+    val cbf = compiled()
+    cbf.asInstanceOf[CanBuildFrom[T, E, T]]
+  }
+  
+}