You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/01/19 00:29:16 UTC
[1/4] flink git commit: [FLINK-1399] Add support for registering
Serializers with Kryo
Repository: flink
Updated Branches:
refs/heads/master d3072ba57 -> 29c54a20d
[FLINK-1399] Add support for registering Serializers with Kryo
This closes #305
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2d4ed15e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2d4ed15e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2d4ed15e
Branch: refs/heads/master
Commit: 2d4ed15e8d2aa1a1fcd7001fccea2ee70cc8a8b0
Parents: d3072ba
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Tue Jan 13 16:13:55 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Sun Jan 18 13:26:04 2015 -0800
----------------------------------------------------------------------
.../flink/api/java/ExecutionEnvironment.java | 18 +++++++++
.../java/typeutils/runtime/KryoSerializer.java | 42 +++++++++++++++++++-
.../flink/api/scala/ExecutionEnvironment.scala | 18 +++++++++
3 files changed, 76 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2d4ed15e/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index f7057ef..1753af8 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -28,6 +28,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
+import com.esotericsoftware.kryo.Serializer;
import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
@@ -54,6 +55,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.api.java.typeutils.ValueTypeInfo;
+import org.apache.flink.api.java.typeutils.runtime.KryoSerializer;
import org.apache.flink.core.fs.Path;
import org.apache.flink.types.StringValue;
import org.apache.flink.util.NumberSequenceIterator;
@@ -198,6 +200,22 @@ public abstract class ExecutionEnvironment {
public String getIdString() {
return this.executionId.toString();
}
+
+ /**
+ * Registers the given Serializer as a default serializer for the given class at the
+ * {@link org.apache.flink.api.java.typeutils.runtime.KryoSerializer}.
+ */
+ public void addDefaultKryoSerializer(Class<?> clazz, Serializer<?> serializer) {
+ KryoSerializer.addDefaultSerializer(clazz, serializer);
+ }
+
+ /**
+ * Registers the given Serializer as a default serializer for the given class at the
+ * {@link org.apache.flink.api.java.typeutils.runtime.KryoSerializer}.
+ */
+ public void addDefaultKryoSerializer(Class<?> clazz, Class<? extends Serializer<?>> serializer) {
+ KryoSerializer.addDefaultSerializer(clazz, serializer);
+ }
// --------------------------------------------------------------------------------------------
// Data set creations
http://git-wip-us.apache.org/repos/asf/flink/blob/2d4ed15e/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
index 0a88606..0a5c4d9 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.api.java.typeutils.runtime;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.KryoException;
+import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
@@ -33,9 +34,17 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
public class KryoSerializer<T> extends TypeSerializer<T> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 3L;
+
+ private static Map<Class<?>, Serializer<?>> staticRegisteredSerializers = new HashMap<Class<?>, Serializer<?>>();
+ private static Map<Class<?>, Class<? extends Serializer<?>>> staticRegisteredSerializersClasses = new HashMap<Class<?>, Class<? extends Serializer<?>>>();
+
+ private static Map<Class<?>, Serializer<?>> registeredSerializers;
+ private static Map<Class<?>, Class<? extends Serializer<?>>> registeredSerializersClasses;
private final Class<T> type;
@@ -53,6 +62,9 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
throw new NullPointerException("Type class cannot be null.");
}
this.type = type;
+
+ this.registeredSerializers = staticRegisteredSerializers;
+ this.registeredSerializersClasses = staticRegisteredSerializersClasses;
}
@@ -182,12 +194,38 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
if (this.kryo == null) {
this.kryo = new ScalaKryoInstantiator().newKryo();
this.kryo.addDefaultSerializer(Throwable.class, new JavaSerializer());
+
+ for (Map.Entry<Class<?>, Serializer<?>> e: registeredSerializers.entrySet()) {
+ this.kryo.addDefaultSerializer(e.getKey(), e.getValue());
+ }
+
+ for (Map.Entry<Class<?>, Class<? extends Serializer<?>>> e: registeredSerializersClasses.entrySet()) {
+ this.kryo.addDefaultSerializer(e.getKey(), e.getValue());
+ }
+
this.kryo.setRegistrationRequired(false);
this.kryo.register(type);
+
this.kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
}
}
-
+
+ /**
+ * Registers the given Serializer as a default serializer for the given class at the Kryo
+ * instance.
+ */
+ public static void addDefaultSerializer(Class<?> clazz, Serializer<?> serializer) {
+ staticRegisteredSerializers.put(clazz, serializer);
+ }
+
+ /**
+ * Registers the given Serializer as a default serializer for the given class at the Kryo
+ * instance.
+ */
+ public static void addDefaultSerializer(Class<?> clazz, Class<? extends Serializer<?>> serializer) {
+ staticRegisteredSerializersClasses.put(clazz, serializer);
+ }
+
// --------------------------------------------------------------------------------------------
// for testing
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2d4ed15e/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
index db2e864..565b9b1 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
@@ -19,11 +19,13 @@ package org.apache.flink.api.scala
import java.util.UUID
+import com.esotericsoftware.kryo.Serializer
import org.apache.commons.lang3.Validate
import org.apache.flink.api.common.{ExecutionConfig, JobExecutionResult}
import org.apache.flink.api.java.io._
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.java.typeutils.runtime.KryoSerializer
import org.apache.flink.api.java.typeutils.{ValueTypeInfo, TupleTypeInfoBase}
import org.apache.flink.api.scala.operators.ScalaCsvInputFormat
import org.apache.flink.core.fs.Path
@@ -122,6 +124,22 @@ class ExecutionEnvironment(javaEnv: JavaEnv) {
}
/**
+ * Registers the given Serializer as a default serializer for the given class at the
+ * [[KryoSerializer]].
+ */
+ def addDefaultKryoSerializer(clazz: Class[_], serializer: Serializer[_]): Unit = {
+ javaEnv.addDefaultKryoSerializer(clazz, serializer)
+ }
+
+ /**
+ * Registers the given Serializer as a default serializer for the given class at the
+ * [[KryoSerializer]]
+ */
+ def addDefaultKryoSerializer(clazz: Class[_], serializer: Class[_ <: Serializer[_]]) {
+ javaEnv.addDefaultKryoSerializer(clazz, serializer)
+ }
+
+ /**
* Creates a DataSet of Strings produced by reading the given file line wise.
*
* @param filePath The path of the file, as a URI (e.g., "file:///some/local/file" or
[4/4] flink git commit: [FLINK-1416] Fix dependency versions in
quickstart archetypes
Posted by se...@apache.org.
[FLINK-1416] Fix dependency versions in quickstart archetypes
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/29c54a20
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/29c54a20
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/29c54a20
Branch: refs/heads/master
Commit: 29c54a20dd89104ef5832c972ac10bbeed958ee1
Parents: 3b58509
Author: Stephan Ewen <se...@apache.org>
Authored: Sun Jan 18 15:24:15 2015 -0800
Committer: Stephan Ewen <se...@apache.org>
Committed: Sun Jan 18 15:24:15 2015 -0800
----------------------------------------------------------------------
.../src/main/resources/archetype-resources/pom.xml | 4 ++--
.../src/main/resources/archetype-resources/pom.xml | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/29c54a20/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 86444c2..adc6ab5 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -57,12 +57,12 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-core</artifactId>
- <version>0.8-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-connectors</artifactId>
- <version>0.8-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/29c54a20/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 7ce030d..49036e8 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -57,7 +57,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala</artifactId>
- <version>0.8-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
[3/4] flink git commit: [tests] Make dates in test data agnostic to
time zones
Posted by se...@apache.org.
[tests] Make dates in test data agnostic to time zones
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3b585094
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3b585094
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3b585094
Branch: refs/heads/master
Commit: 3b58509465450e1bd80508f28e23c5a6c27972c0
Parents: a0b6af2
Author: Stephan Ewen <se...@apache.org>
Authored: Sun Jan 18 14:35:44 2015 -0800
Committer: Stephan Ewen <se...@apache.org>
Committed: Sun Jan 18 14:35:44 2015 -0800
----------------------------------------------------------------------
.../test/javaApiOperators/util/CollectionDataSets.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/3b585094/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
index aee0031..54f088a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
@@ -25,6 +25,7 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
+import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -38,6 +39,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.hadoop.io.IntWritable;
+
import scala.math.BigInt;
/**
@@ -614,7 +616,10 @@ public class CollectionDataSets {
pwc1.bigInt = BigInteger.valueOf(Long.MAX_VALUE).multiply(BigInteger.TEN);
pwc1.scalaBigInt = BigInt.int2bigInt(10);
pwc1.bigDecimalKeepItNull = null;
- pwc1.sqlDate = new java.sql.Date(2000000000000L); // 2033 ;)
+
+ // use calendar to make it stable across time zones
+ GregorianCalendar gcl1 = new GregorianCalendar(2033, 04, 18);
+ pwc1.sqlDate = new java.sql.Date(gcl1.getTimeInMillis());
pwc1.mixed = new ArrayList<Object>();
Map<String, Integer> map = new HashMap<String, Integer>();
map.put("someKey", 1); // map.put("anotherKey", 2); map.put("third", 3);
@@ -628,7 +633,9 @@ public class CollectionDataSets {
pwc2.bigInt = BigInteger.valueOf(Long.MAX_VALUE).multiply(BigInteger.TEN);
pwc2.scalaBigInt = BigInt.int2bigInt(31104000);
pwc2.bigDecimalKeepItNull = null;
- pwc2.sqlDate = new java.sql.Date(200000000000L); // 1976
+
+ GregorianCalendar gcl2 = new GregorianCalendar(1976, 4, 3);
+ pwc2.sqlDate = new java.sql.Date(gcl2.getTimeInMillis()); // 1976
data.add(pwc1);
[2/4] flink git commit: [FLINK-1399] Fix kryo registration of types
and make sure that tags are assigned for registered types.
Posted by se...@apache.org.
[FLINK-1399] Fix kryo registration of types and make sure that tags are assigned for registered types.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a0b6af20
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a0b6af20
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a0b6af20
Branch: refs/heads/master
Commit: a0b6af20beed6d12d0f33f6f58f323b20cc12961
Parents: 2d4ed15
Author: Stephan Ewen <se...@apache.org>
Authored: Sun Jan 18 13:04:07 2015 -0800
Committer: Stephan Ewen <se...@apache.org>
Committed: Sun Jan 18 13:59:39 2015 -0800
----------------------------------------------------------------------
.../flink/api/java/ExecutionEnvironment.java | 50 ++++++-
.../java/typeutils/runtime/KryoSerializer.java | 147 ++++++++++++++++---
.../flink/api/scala/ExecutionEnvironment.scala | 19 ++-
3 files changed, 184 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a0b6af20/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index 1753af8..77fed97 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.UUID;
import com.esotericsoftware.kryo.Serializer;
+
import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
@@ -201,20 +202,57 @@ public abstract class ExecutionEnvironment {
return this.executionId.toString();
}
+ // --------------------------------------------------------------------------------------------
+ // Registry for types and serializers
+ // --------------------------------------------------------------------------------------------
+
/**
- * Registers the given Serializer as a default serializer for the given class at the
+ * Registers the given Serializer as a default serializer for the given type at the
* {@link org.apache.flink.api.java.typeutils.runtime.KryoSerializer}.
+ *
+ * Note that the serializer instance must be serializable (as defined by java.io.Serializable),
+ * because it may be distributed to the worker nodes by java serialization.
+ *
+ * @param type The class of the types serialized with the given serializer.
+ * @param serializer The serializer to use.
*/
- public void addDefaultKryoSerializer(Class<?> clazz, Serializer<?> serializer) {
- KryoSerializer.addDefaultSerializer(clazz, serializer);
+ public void registerKryoSerializer(Class<?> type, Serializer<?> serializer) {
+ if (type == null || serializer == null) {
+ throw new NullPointerException("Cannot register null class or serializer.");
+ }
+
+ KryoSerializer.registerSerializer(type, serializer);
}
/**
- * Registers the given Serializer as a default serializer for the given class at the
+ * Registers the given Serializer via its class as a serializer for the given type at the
* {@link org.apache.flink.api.java.typeutils.runtime.KryoSerializer}.
+ *
+ * @param type The class of the types serialized with the given serializer.
+ * @param serializerClass The class of the serializer to use.
*/
- public void addDefaultKryoSerializer(Class<?> clazz, Class<? extends Serializer<?>> serializer) {
- KryoSerializer.addDefaultSerializer(clazz, serializer);
+ public void registerKryoSerializer(Class<?> type, Class<? extends Serializer<?>> serializerClass) {
+ if (type == null || serializerClass == null) {
+ throw new NullPointerException("Cannot register null class or serializer.");
+ }
+
+ KryoSerializer.registerSerializer(type, serializerClass);
+ }
+
+ /**
+ * Registers the given type with the serialization stack. If the type is eventually
+ * serialized as a POJO, then the type is registered with the POJO serializer. If the
+ * type ends up being serialized with Kryo, then it will be registered at Kryo to make
+ * sure that only tags are written.
+ *
+ * @param type The class of the type to register.
+ */
+ public void registerType(Class<?> type) {
+ if (type == null) {
+ throw new NullPointerException("Cannot register null type class.");
+ }
+
+ KryoSerializer.registerType(type);
}
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a0b6af20/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
index 0a5c4d9..bec5d59 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/KryoSerializer.java
@@ -21,6 +21,7 @@ package org.apache.flink.api.java.typeutils.runtime;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.KryoException;
import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.factories.ReflectionSerializerFactory;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
@@ -34,19 +35,40 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
-
+import java.util.Set;
+
+/**
+ * A type serializer that serializes its type using the Kryo serialization
+ * framework (https://github.com/EsotericSoftware/kryo).
+ *
+ * This serializer is intended as a fallback serializer for the cases that are
+ * not covered by the basic types, tuples, and POJOs.
+ *
+ * @param <T> The type to be serialized.
+ */
public class KryoSerializer<T> extends TypeSerializer<T> {
+
private static final long serialVersionUID = 3L;
private static Map<Class<?>, Serializer<?>> staticRegisteredSerializers = new HashMap<Class<?>, Serializer<?>>();
private static Map<Class<?>, Class<? extends Serializer<?>>> staticRegisteredSerializersClasses = new HashMap<Class<?>, Class<? extends Serializer<?>>>();
-
- private static Map<Class<?>, Serializer<?>> registeredSerializers;
- private static Map<Class<?>, Class<? extends Serializer<?>>> registeredSerializersClasses;
+
+ private static Set<Class<?>> staticRegisteredTypes = new HashSet<Class<?>>();
+
+ // ------------------------------------------------------------------------
+
+ private final Map<Class<?>, Serializer<?>> registeredSerializers;
+ private final Map<Class<?>, Class<? extends Serializer<?>>> registeredSerializersClasses;
+ private final Set<Class<?>> registeredTypes;
private final Class<T> type;
+
+ // ------------------------------------------------------------------------
+ // The fields below are lazily initialized after de-serialization
private transient Kryo kryo;
private transient T copyInstance;
@@ -56,6 +78,8 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
private transient Input input;
private transient Output output;
+
+ // ------------------------------------------------------------------------
public KryoSerializer(Class<T> type){
if(type == null){
@@ -63,10 +87,26 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
}
this.type = type;
- this.registeredSerializers = staticRegisteredSerializers;
- this.registeredSerializersClasses = staticRegisteredSerializersClasses;
+ // create copies of the statically registered serializers
+ // we use static synchronization to safeguard against concurrent use
+ // of the static collections.
+ synchronized (KryoSerializer.class) {
+ this.registeredSerializers = staticRegisteredSerializers.isEmpty() ?
+ Collections.<Class<?>, Serializer<?>>emptyMap() :
+ new HashMap<Class<?>, Serializer<?>>(staticRegisteredSerializers);
+
+ this.registeredSerializersClasses = staticRegisteredSerializersClasses.isEmpty() ?
+ Collections.<Class<?>, Class<? extends Serializer<?>>>emptyMap() :
+ new HashMap<Class<?>, Class<? extends Serializer<?>>>(staticRegisteredSerializersClasses);
+
+ this.registeredTypes = staticRegisteredTypes.isEmpty() ?
+ Collections.<Class<?>>emptySet() :
+ new HashSet<Class<?>>(staticRegisteredTypes);
+ }
+
}
+ // ------------------------------------------------------------------------
@Override
public boolean isImmutableType() {
@@ -193,41 +233,104 @@ public class KryoSerializer<T> extends TypeSerializer<T> {
private void checkKryoInitialized() {
if (this.kryo == null) {
this.kryo = new ScalaKryoInstantiator().newKryo();
- this.kryo.addDefaultSerializer(Throwable.class, new JavaSerializer());
- for (Map.Entry<Class<?>, Serializer<?>> e: registeredSerializers.entrySet()) {
- this.kryo.addDefaultSerializer(e.getKey(), e.getValue());
- }
+ // Throwable and all subclasses should be serialized via java serialization
+ kryo.addDefaultSerializer(Throwable.class, new JavaSerializer());
- for (Map.Entry<Class<?>, Class<? extends Serializer<?>>> e: registeredSerializersClasses.entrySet()) {
- this.kryo.addDefaultSerializer(e.getKey(), e.getValue());
+ // register the type of our class
+ kryo.register(type);
+
+ // register given types. we do this first so that any registration of a
+ // more specific serializer overrides this
+ for (Class<?> type : registeredTypes) {
+ kryo.register(type);
+ }
+
+ // register given serializer classes
+ for (Map.Entry<Class<?>, Class<? extends Serializer<?>>> e : registeredSerializersClasses.entrySet()) {
+ Class<?> typeClass = e.getKey();
+ Class<? extends Serializer<?>> serializerClass = e.getValue();
+
+ Serializer<?> serializer =
+ ReflectionSerializerFactory.makeSerializer(kryo, serializerClass, typeClass);
+ kryo.register(typeClass, serializer);
}
- this.kryo.setRegistrationRequired(false);
- this.kryo.register(type);
-
- this.kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
+ // register given serializers
+ for (Map.Entry<Class<?>, Serializer<?>> e : registeredSerializers.entrySet()) {
+ kryo.register(e.getKey(), e.getValue());
+ }
+
+ kryo.setRegistrationRequired(false);
+ kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
}
}
+
+ // --------------------------------------------------------------------------------------------
+ // For registering custom serializers and types
+ // --------------------------------------------------------------------------------------------
/**
* Registers the given Serializer as a default serializer for the given class at the Kryo
* instance.
+ * Note that the serializer instance must be serializable (as defined by java.io.Serializable),
+ * because it may be distributed to the worker nodes by java serialization.
+ *
+ * @param clazz The class of the types serialized with the given serializer.
+ * @param serializer The serializer to use.
+ * @throws IllegalArgumentException Thrown, if the serializer is not serializable.
*/
- public static void addDefaultSerializer(Class<?> clazz, Serializer<?> serializer) {
- staticRegisteredSerializers.put(clazz, serializer);
+ public static void registerSerializer(Class<?> clazz, Serializer<?> serializer) {
+ if (clazz == null || serializer == null) {
+ throw new NullPointerException("Cannot register null class or serializer.");
+ }
+ if (!(serializer instanceof java.io.Serializable)) {
+ throw new IllegalArgumentException("The serializer instance must be serializable, (for distributing it in the cluster), "
+ + "as defined by java.io.Serializable. For stateless serializers, you can use the "
+ + "'registerSerializer(Class, Class)' method to register the serializer via its class.");
+ }
+
+ synchronized (KryoSerializer.class) {
+ staticRegisteredSerializers.put(clazz, serializer);
+ }
}
/**
- * Registers the given Serializer as a default serializer for the given class at the Kryo
+ * Registers a serializer via its class as a default serializer for the given class at the Kryo
* instance.
+ *
+ * @param clazz The class of the types serialized with the given serializer.
+ * @param serializerClass The serializer to use.
*/
- public static void addDefaultSerializer(Class<?> clazz, Class<? extends Serializer<?>> serializer) {
- staticRegisteredSerializersClasses.put(clazz, serializer);
+ public static void registerSerializer(Class<?> clazz, Class<? extends Serializer<?>> serializerClass) {
+ if (clazz == null || serializerClass == null) {
+ throw new NullPointerException("Cannot register null class or serializer.");
+ }
+
+ synchronized (KryoSerializer.class) {
+ staticRegisteredSerializersClasses.put(clazz, serializerClass);
+ }
+ }
+
+ /**
+ * Registers the given type with Kryo. Registering the type allows Kryo to write abbreviated
+ * name tags, rather than full class names, thereby vastly increasing the serialization
+ * performance in many cases.
+ *
+ * @param type The class of the type to register.
+ */
+ public static void registerType(Class<?> type) {
+ if (type == null) {
+ throw new NullPointerException("Cannot register null type class.");
+ }
+
+ synchronized (KryoSerializer.class) {
+ staticRegisteredTypes.add(type);
+ }
}
// --------------------------------------------------------------------------------------------
- // for testing
+ // For testing
// --------------------------------------------------------------------------------------------
Kryo getKryo() {
http://git-wip-us.apache.org/repos/asf/flink/blob/a0b6af20/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
index 565b9b1..391986d 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/ExecutionEnvironment.scala
@@ -127,16 +127,27 @@ class ExecutionEnvironment(javaEnv: JavaEnv) {
* Registers the given Serializer as a default serializer for the given class at the
* [[KryoSerializer]].
*/
- def addDefaultKryoSerializer(clazz: Class[_], serializer: Serializer[_]): Unit = {
- javaEnv.addDefaultKryoSerializer(clazz, serializer)
+ def registerKryoSerializer(clazz: Class[_], serializer: Serializer[_]): Unit = {
+ javaEnv.registerKryoSerializer(clazz, serializer)
}
/**
* Registers the given Serializer as a default serializer for the given class at the
* [[KryoSerializer]]
*/
- def addDefaultKryoSerializer(clazz: Class[_], serializer: Class[_ <: Serializer[_]]) {
- javaEnv.addDefaultKryoSerializer(clazz, serializer)
+ def registerKryoSerializer(clazz: Class[_], serializer: Class[_ <: Serializer[_]]) {
+ javaEnv.registerKryoSerializer(clazz, serializer)
+ }
+
+ /**
+ * Registers the given type with the serialization stack. If the type is eventually
+ * serialized as a POJO, then the type is registered with the POJO serializer. If the
+ * type ends up being serialized with Kryo, then it will be registered at Kryo to make
+ * sure that only tags are written.
+ *
+ */
+ def registerType(typeClass: Class[_]) {
+ javaEnv.registerType(typeClass)
}
/**