You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2015/11/30 22:41:46 UTC
flink git commit: [FLINK-2972] [JavaAPI] Remove Chill dependency from
flink-java.
Repository: flink
Updated Branches:
refs/heads/master d5a6b13ab -> cd899f3be
[FLINK-2972] [JavaAPI] Remove Chill dependency from flink-java.
This closes #1331
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cd899f3b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cd899f3b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cd899f3b
Branch: refs/heads/master
Commit: cd899f3be39c5f052a2a3e90771111d1a7df1d47
Parents: d5a6b13
Author: Fabian Hueske <fh...@apache.org>
Authored: Tue Nov 3 16:57:49 2015 +0100
Committer: Fabian Hueske <fh...@apache.org>
Committed: Mon Nov 30 20:31:42 2015 +0100
----------------------------------------------------------------------
flink-core/pom.xml | 6 -----
flink-java/pom.xml | 17 +++++++------
.../typeutils/runtime/kryo/KryoSerializer.java | 25 ++++++++++++++++++--
flink-runtime/pom.xml | 12 ++++++++++
flink-scala/pom.xml | 2 +-
5 files changed, 46 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index 6763011..f91608e 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -53,12 +53,6 @@ under the License.
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
- <artifactId>chill_${scala.binary.version}</artifactId>
- <version>${chill.version}</version>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index 4b8e24c..0b33de1 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -65,13 +65,7 @@ under the License.
<dependency>
<groupId>com.twitter</groupId>
- <artifactId>chill_${scala.binary.version}</artifactId>
- <version>${chill.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>chill-avro_${scala.binary.version}</artifactId>
+ <artifactId>chill-java</artifactId>
<version>${chill.version}</version>
</dependency>
@@ -110,6 +104,15 @@ under the License.
<type>test-jar</type>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>chill_${scala.binary.version}</artifactId>
+ <version>${chill.version}</version>
+ <!-- For execution, Chill is added to the classpath through flink-runtime. -->
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<!-- Because flink-scala and flink-avro uses it in tests -->
http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
index 8549e26..b90901c 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/kryo/KryoSerializer.java
@@ -26,7 +26,6 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
import com.google.common.base.Preconditions;
-import com.twitter.chill.ScalaKryoInstantiator;
import org.apache.avro.generic.GenericData;
import org.apache.flink.api.common.ExecutionConfig;
@@ -42,6 +41,8 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -281,9 +282,29 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
// --------------------------------------------------------------------------------------------
+ private Kryo getKryoInstance() {
+
+ try {
+ // check if ScalaKryoInstantiator is in class path (coming from Twitter's Chill library).
+ // This will be true if Flink's Scala API is used.
+ Class<?> chillInstantiatorClazz = Class.forName("com.twitter.chill.ScalaKryoInstantiator");
+ Object chillInstantiator = chillInstantiatorClazz.newInstance();
+
+ // obtain a Kryo instance through Twitter Chill
+ Method m = chillInstantiatorClazz.getMethod("newKryo");
+ return (Kryo) m.invoke(chillInstantiator);
+ }
+ catch(ClassNotFoundException | InstantiationException | NoSuchMethodException |
+ IllegalAccessException | InvocationTargetException e ) {
+
+ // Chill must be in the classpath. It is added as a dependency to flink-runtime.
+ throw new RuntimeException("Could not instantiate Kryo instance from Chill.", e);
+ }
+ }
+
private void checkKryoInitialized() {
if (this.kryo == null) {
- this.kryo = new ScalaKryoInstantiator().newKryo();
+ this.kryo = getKryoInstance();
// Throwable and all subclasses should be serialized via java serialization
kryo.addDefaultSerializer(Throwable.class, new JavaSerializer());
http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index a1e0f75..c3093da 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -180,6 +180,18 @@ under the License.
<artifactId>zookeeper</artifactId>
</dependency>
+ <!--
+ The KryoSerializer dynamically loads Kryo instances via Chill and requires that Chill
+ is in the classpath. Because we do not want to have transitive Scala dependencies
+ in Flink's API modules (such as flink-java) due to Chill, Chill is added to flink-runtime
+ to ensure that Chill is always present in the classpath.
+ -->
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>chill_${scala.binary.version}</artifactId>
+ <version>${chill.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-curator-recipes</artifactId>
http://git-wip-us.apache.org/repos/asf/flink/blob/cd899f3b/flink-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 32b5948..d30dad6 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -69,7 +69,7 @@ under the License.
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
- <version>${asm.version}</version>
+ <version>${asm.version}</version>
</dependency>
<dependency>