You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ma...@apache.org on 2015/06/24 21:24:31 UTC
git commit: updated refs/heads/trunk to add1d4f
Repository: giraph
Updated Branches:
refs/heads/trunk ffed230ba -> add1d4f07
GIRAPH-1017: Add support for ImmutableMap in Kryo
Summary: Trying to serialize ImmutableMap currently throws an exception - we should add a support for it.
Test Plan: Added a test, verified that app which was failing without the change passes now
Reviewers: ikabiljo
Differential Revision: https://reviews.facebook.net/D40575
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/add1d4f0
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/add1d4f0
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/add1d4f0
Branch: refs/heads/trunk
Commit: add1d4f07c925b8a9044cb3aa5bb4abdeaf49fc7
Parents: ffed230
Author: Maja Kabiljo <ma...@fb.com>
Authored: Tue Jun 23 16:48:55 2015 -0700
Committer: Maja Kabiljo <ma...@fb.com>
Committed: Wed Jun 24 12:23:35 2015 -0700
----------------------------------------------------------------------
.../apache/giraph/writable/kryo/HadoopKryo.java | 31 ++++++++++-----
.../serializers/ImmutableMapSerializer.java | 40 ++++++++++++++++++++
.../giraph/writable/kryo/KryoWritableTest.java | 16 ++++++++
pom.xml | 2 +-
4 files changed, 79 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/add1d4f0/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
index b4f2bfa..478a33b 100644
--- a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
+++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/HadoopKryo.java
@@ -36,6 +36,7 @@ import org.apache.giraph.writable.kryo.serializers.ArraysAsListSerializer;
import org.apache.giraph.writable.kryo.serializers.CollectionsNCopiesSerializer;
import org.apache.giraph.writable.kryo.serializers.DirectWritableSerializer;
import org.apache.giraph.writable.kryo.serializers.FastUtilSerializer;
+import org.apache.giraph.writable.kryo.serializers.ImmutableMapSerializer;
import org.apache.giraph.writable.kryo.serializers.ReusableFieldSerializer;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -229,15 +230,11 @@ public class HadoopKryo extends Kryo {
ImmutableListSerializer.registerSerializers(kryo);
- // TODO move guava version to 18.0, and remove this fix:
- try {
- kryo.register(
- Class.forName("com.google.common.collect.RegularImmutableList"),
- new ImmutableListSerializer());
- } catch (ClassNotFoundException e) {
- throw new IllegalStateException(
- "Guava has RegularImmutableList missing", e);
- }
+ registerSerializer(kryo, "com.google.common.collect.RegularImmutableMap",
+ new ImmutableMapSerializer());
+ registerSerializer(kryo,
+ "com.google.common.collect.SingletonImmutableBiMap",
+ new ImmutableMapSerializer());
// There are many fastutil classes, register them at the end,
// so they don't use up small registration numbers
@@ -298,6 +295,22 @@ public class HadoopKryo extends Kryo {
}
/**
+ * Register serializer for class with class name
+ *
+ * @param kryo HadoopKryo
+ * @param className Name of the class for which to register serializer
+ * @param serializer Serializer to use
+ */
+ private static void registerSerializer(HadoopKryo kryo, String className,
+ Serializer serializer) {
+ try {
+ kryo.register(Class.forName(className), serializer);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalStateException("Class " + className + " is missing", e);
+ }
+ }
+
+ /**
* Initialize reusable objects for reading from given DataInput.
*
* @param in Input stream
http://git-wip-us.apache.org/repos/asf/giraph/blob/add1d4f0/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java
new file mode 100644
index 0000000..8dd2d1c
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/writable/kryo/serializers/ImmutableMapSerializer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph.writable.kryo.serializers;
+
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.serializers.MapSerializer;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Serializer for {@link ImmutableMap}
+ */
+public class ImmutableMapSerializer extends MapSerializer {
+ @Override
+ public Map read(Kryo kryo, Input input, Class<Map> type) {
+ Map map = super.read(kryo, input,
+ (Class<Map>) ((Object) HashMap.class));
+ return ImmutableMap.copyOf(map);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/add1d4f0/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableTest.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableTest.java b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableTest.java
index 3898b82..19c7ee4 100644
--- a/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableTest.java
+++ b/giraph-core/src/test/java/org/apache/giraph/writable/kryo/KryoWritableTest.java
@@ -18,6 +18,8 @@
package org.apache.giraph.writable.kryo;
import static org.junit.Assert.assertEquals;
+
+import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import java.util.Arrays;
@@ -29,6 +31,7 @@ import org.apache.hadoop.io.LongWritable;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.collect.ImmutableMap;
/**
@@ -192,4 +195,17 @@ public class KryoWritableTest {
assertEquals(5, res.value1.get());
Assert.assertTrue(res == res.value2);
}
+
+ @Test
+ public void testKryoImmutableMap() throws Exception {
+ Long2IntOpenHashMap map = new Long2IntOpenHashMap();
+ map.put(1, 2);
+ map.put(10, 20);
+ ImmutableMap<Long, Integer> copy =
+ WritableUtils.createCopy(
+ new KryoWritableWrapper<>(ImmutableMap.copyOf(map))).get();
+ Assert.assertEquals(2, copy.size());
+ Assert.assertEquals(2, copy.get(1L).intValue());
+ Assert.assertEquals(20, copy.get(10L).intValue());
+ }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/add1d4f0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c32dfc7..eb4f2f7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -297,7 +297,7 @@ under the License.
<dep.fasterxml-jackson.version>2.1.2</dep.fasterxml-jackson.version>
<dep.fastutil.version>6.5.4</dep.fastutil.version>
<dep.google.findbugs.version>2.0.2</dep.google.findbugs.version>
- <dep.guava.version>14.0.1</dep.guava.version>
+ <dep.guava.version>18.0</dep.guava.version>
<dep.hbase.version>0.94.16</dep.hbase.version>
<dep.hcatalog.version>0.5.0-incubating</dep.hcatalog.version>
<dep.hive.version>0.11.0</dep.hive.version>