You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by di...@apache.org on 2018/05/31 18:08:04 UTC
git commit: updated refs/heads/trunk to 0c1e2eb
Repository: giraph
Updated Branches:
refs/heads/trunk 505c98d9a -> 0c1e2ebce
GIRAPH-1195
closes #76
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/0c1e2ebc
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/0c1e2ebc
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/0c1e2ebc
Branch: refs/heads/trunk
Commit: 0c1e2ebce6e4f6e1a47d4dec5c0b8475433ef8df
Parents: 505c98d
Author: Dionysios Logothetis <dl...@gmail.com>
Authored: Thu May 31 11:07:33 2018 -0700
Committer: Dionysios Logothetis <di...@fb.com>
Committed: Thu May 31 11:07:33 2018 -0700
----------------------------------------------------------------------
.../apache/giraph/writable/kryo/HadoopKryo.java | 14 ++---
.../ImmutableBiMapSerializerUtils.java | 60 ++++++++++++++++++++
.../serializers/ImmutableMapSerializer.java | 40 -------------
.../writable/kryo/KryoWritableWrapperTest.java | 38 +++++++++++++
pom.xml | 12 +++-
5 files changed, 113 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
index 2713316..5d51a3f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
+++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
@@ -27,6 +27,7 @@ import java.util.Map.Entry;
import java.util.Random;
import com.esotericsoftware.kryo.util.DefaultClassResolver;
+import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer;
import org.apache.giraph.conf.GiraphConfigurationSettable;
import com.esotericsoftware.kryo.ClassResolver;
import com.esotericsoftware.kryo.ReferenceResolver;
@@ -39,7 +40,7 @@ import org.apache.giraph.writable.kryo.serializers.ArraysAsListSerializer;
import org.apache.giraph.writable.kryo.serializers.CollectionsNCopiesSerializer;
import org.apache.giraph.writable.kryo.serializers.DirectWritableSerializer;
import org.apache.giraph.writable.kryo.serializers.FastUtilSerializer;
-import org.apache.giraph.writable.kryo.serializers.ImmutableMapSerializer;
+import org.apache.giraph.writable.kryo.serializers.ImmutableBiMapSerializerUtils;
import org.apache.giraph.writable.kryo.serializers.ReusableFieldSerializer;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -322,7 +323,8 @@ public class HadoopKryo extends Kryo {
if (minor >= '8') {
try {
kryo.register(Class.forName("java.lang.invoke.SerializedLambda"));
- kryo.register(Class.forName("com.esotericsoftware.kryo.Kryo$Closure"),
+ kryo.register(Class.forName(
+ "com.esotericsoftware.kryo.serializers.ClosureSerializer$Closure"),
new ClosureSerializer());
} catch (ClassNotFoundException e) {
throw new IllegalStateException(
@@ -336,12 +338,8 @@ public class HadoopKryo extends Kryo {
new CollectionsNCopiesSerializer());
ImmutableListSerializer.registerSerializers(kryo);
-
- registerSerializer(kryo, "com.google.common.collect.RegularImmutableMap",
- new ImmutableMapSerializer());
- registerSerializer(kryo,
- "com.google.common.collect.SingletonImmutableBiMap",
- new ImmutableMapSerializer());
+ ImmutableMapSerializer.registerSerializers(kryo);
+ ImmutableBiMapSerializerUtils.registerSerializers(kryo);
// There are many fastutil classes, register them at the end,
// so they don't use up small registration numbers
http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java
new file mode 100644
index 0000000..6e66b47
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableBiMapSerializerUtils.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.writable.kryo.serializers;
+
+
+import com.esotericsoftware.kryo.Kryo;
+import com.google.common.collect.ImmutableBiMap;
+import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer;
+
+/**
+ * Helper class used to register serializers for the different classes
+ * return by the {@link ImmutableBiMap} methods.
+ */
+public class ImmutableBiMapSerializerUtils {
+
+ /**
+ * Private default constructor
+ */
+ private ImmutableBiMapSerializerUtils() {
+ }
+
+ /**
+ * Registers different {@link ImmutableBiMap} implementations with Kryo.
+ * @param kryo {@link Kryo} instance to register class with.
+ */
+ public static void registerSerializers(Kryo kryo) {
+ ImmutableMapSerializer serializer = new ImmutableMapSerializer();
+ kryo.register(ImmutableBiMap.class, serializer);
+ kryo.register(ImmutableBiMap.of().getClass(), serializer);
+ Object o1 = new Object();
+ Object o2 = new Object();
+ Object o3 = new Object();
+ Object o4 = new Object();
+ Object o5 = new Object();
+ kryo.register(ImmutableBiMap.of(o1, o1).getClass(), serializer);
+ kryo.register(ImmutableBiMap.of(o1, o1, o2, o2).getClass(), serializer);
+ kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3)
+ .getClass(), serializer);
+ kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4)
+ .getClass(), serializer);
+ kryo.register(ImmutableBiMap.of(o1, o1, o2, o2, o3, o3, o4, o4, o5, o5)
+ .getClass(), serializer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java
deleted file mode 100644
index 8dd2d1c..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.writable.kryo.serializers;
-
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.serializers.MapSerializer;
-import com.google.common.collect.ImmutableMap;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Serializer for {@link ImmutableMap}
- */
-public class ImmutableMapSerializer extends MapSerializer {
- @Override
- public Map read(Kryo kryo, Input input, Class<Map> type) {
- Map map = super.read(kryo, input,
- (Class<Map>) ((Object) HashMap.class));
- return ImmutableMap.copyOf(map);
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java
index 6e0eb2f..9129a4e 100644
--- a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java
+++ b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableWrapperTest.java
@@ -25,8 +25,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Random;
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableMap;
import org.apache.giraph.conf.GiraphConfiguration;
import org.apache.giraph.conf.GiraphConfigurationSettable;
import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
@@ -171,6 +174,41 @@ public class KryoWritableWrapperTest {
}
@Test
+ public void testImmutableMapSerialization() throws IOException {
+ Map original = ImmutableMap.of("x", "y", "y", "z");
+ Map copy = kryoSerDeser(original);
+ assertEquals(original, copy);
+ }
+
+ @Test
+ public void testImmutableMapSinglePairSerialization() throws IOException {
+ Map original = ImmutableMap.of("x", "y");
+ Map copy = kryoSerDeser(original);
+ assertEquals(original, copy);
+ }
+
+ @Test
+ public void testImmutableBiMap() throws IOException {
+ Map original = ImmutableBiMap.of("x", "y", "z", "w");
+ Map copy = kryoSerDeser(original);
+ assertEquals(original, copy);
+ }
+
+ @Test
+ public void testSingletonImmutableBiMapSerialization() throws IOException {
+ Map original = ImmutableBiMap.of("x", "y");
+ Map copy = kryoSerDeser(original);
+ assertEquals(original, copy);
+ }
+
+ @Test
+ public void testEmptyImmutableBiMap() throws IOException {
+ Map original = ImmutableBiMap.of();
+ Map copy = kryoSerDeser(original);
+ assertEquals(original, copy);
+ }
+
+ @Test
public void testFastutilSet() throws ClassNotFoundException, IOException {
LongOpenHashSet set = new LongOpenHashSet();
set.add(6);
http://git-wip-us.apache.org/repos/asf/giraph/blob/0c1e2ebc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fd752fb..1522fd4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -343,15 +343,16 @@ under the License.
<dep.json.version>20160810</dep.json.version>
<dep.junit.version>4.11</dep.junit.version>
<dep.jython.version>2.5.3</dep.jython.version>
- <dep.kryo.version>3.0.1</dep.kryo.version>
- <dep.kryo-serializers.version>0.29</dep.kryo-serializers.version>
+ <dep.kryo.version>4.0.0</dep.kryo.version>
+ <dep.reflectasm.version>1.11.3</dep.reflectasm.version>
+ <dep.kryo-serializers.version>0.42</dep.kryo-serializers.version>
<dep.libthrift.version>0.9.0</dep.libthrift.version>
<dep.log4j.version>1.2.17</dep.log4j.version>
<dep.mockito.version>1.9.5</dep.mockito.version>
<!-- note: old version of netty is required by hadoop_facebook for tests to succeed -->
<dep.netty.version>4.0.14.Final</dep.netty.version>
<dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version>
- <dep.objenesis.version>2.1</dep.objenesis.version>
+ <dep.objenesis.version>2.2</dep.objenesis.version>
<dep.openhft-compiler.version>2.2.1</dep.openhft-compiler.version>
<dep.paranamer.version>2.5.2</dep.paranamer.version>
<dep.slf4j.version>1.7.6</dep.slf4j.version>
@@ -2050,6 +2051,11 @@ under the License.
<version>${dep.kryo.version}</version>
</dependency>
<dependency>
+ <groupId>com.esotericsoftware.reflectasm</groupId>
+ <artifactId>reflectasm</artifactId>
+ <version>${dep.reflectasm.version}</version>
+ </dependency>
+ <dependency>
<groupId>de.javakaffee</groupId>
<artifactId>kryo-serializers</artifactId>
<version>${dep.kryo-serializers.version}</version>