You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by up...@apache.org on 2018/04/13 19:01:51 UTC
[geode] branch feature/transcoding_experiments updated: Adding a
value serializer that compresses values
This is an automated email from the ASF dual-hosted git repository.
upthewaterspout pushed a commit to branch feature/transcoding_experiments
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/feature/transcoding_experiments by this push:
new e2c2ebe Adding a value serializer that compresses values
e2c2ebe is described below
commit e2c2ebef383e95995225240a7c9ff82ba0518266
Author: Dan Smith <up...@apache.org>
AuthorDate: Fri Apr 13 12:01:26 2018 -0700
Adding a value serializer that compresses values
---
.../CompressingProtobufStructSerializer.java | 50 +++++++++++++
.../serialization/ProtobufStructSerializer.java | 2 +-
...he.geode.protocol.serialization.ValueSerializer | 3 +-
.../CompressingProtobufStructSerializerTest.java | 84 ++++++++++++++++++++++
.../ProtobufStructSerializerTest.java | 11 ++-
5 files changed, 141 insertions(+), 9 deletions(-)
diff --git a/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/CompressingProtobufStructSerializer.java b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/CompressingProtobufStructSerializer.java
new file mode 100644
index 0000000..c807062
--- /dev/null
+++ b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/CompressingProtobufStructSerializer.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.protocol.serialization;
+
+import java.io.IOException;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.UnsafeByteOperations;
+import org.iq80.snappy.Snappy;
+
+import org.apache.geode.cache.Cache;
+import org.apache.geode.compression.SnappyCompressor;
+import org.apache.geode.internal.protocol.protobuf.v1.Struct;
+
+public class CompressingProtobufStructSerializer implements ValueSerializer {
+ private final ProtobufStructSerializer delegate = new ProtobufStructSerializer();
+
+ @Override
+ public ByteString serialize(Object object) throws IOException {
+ Struct uncompressed = delegate.serializeStruct(object);
+ byte[] compressed = Snappy.compress(uncompressed.toByteArray());
+ return UnsafeByteOperations.unsafeWrap(compressed);
+ }
+
+ @Override
+ public Object deserialize(ByteString bytes) throws IOException, ClassNotFoundException {
+ byte[] compressed = bytes.toByteArray();
+ byte[] uncompressed = Snappy.uncompress(compressed, 0, compressed.length);
+ ByteString byteString = UnsafeByteOperations.unsafeWrap(uncompressed);
+ return delegate.deserialize(byteString);
+ }
+
+ @Override
+ public void init(Cache cache) {
+ delegate.init(cache);
+
+ }
+}
diff --git a/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java
index 16beddf..4c75ced 100644
--- a/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java
+++ b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java
@@ -41,7 +41,7 @@ public class ProtobufStructSerializer implements ValueSerializer {
return serializeStruct(object).toByteString();
}
- private Struct serializeStruct(Object object) {
+ Struct serializeStruct(Object object) {
PdxInstance pdxInstance = (PdxInstance) object;
diff --git a/geode-protobuf/src/main/resources/META-INF/services/org.apache.geode.protocol.serialization.ValueSerializer b/geode-protobuf/src/main/resources/META-INF/services/org.apache.geode.protocol.serialization.ValueSerializer
index 2c4c837..718b795 100644
--- a/geode-protobuf/src/main/resources/META-INF/services/org.apache.geode.protocol.serialization.ValueSerializer
+++ b/geode-protobuf/src/main/resources/META-INF/services/org.apache.geode.protocol.serialization.ValueSerializer
@@ -1 +1,2 @@
-org.apache.geode.protocol.serialization.ProtobufStructSerializer
\ No newline at end of file
+org.apache.geode.protocol.serialization.ProtobufStructSerializer
+org.apache.geode.protocol.serialization.CompressingProtobufStructSerializer
\ No newline at end of file
diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/CompressingProtobufStructSerializerTest.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/CompressingProtobufStructSerializerTest.java
new file mode 100644
index 0000000..2a65b12
--- /dev/null
+++ b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/CompressingProtobufStructSerializerTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.protocol.serialization;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.protobuf.ByteString;
+import com.pholser.junit.quickcheck.From;
+import com.pholser.junit.quickcheck.Property;
+import com.pholser.junit.quickcheck.When;
+import com.pholser.junit.quickcheck.runner.JUnitQuickcheck;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import org.apache.geode.cache.Cache;
+import org.apache.geode.cache.CacheFactory;
+import org.apache.geode.distributed.ConfigurationProperties;
+import org.apache.geode.internal.protocol.protobuf.v1.BasicTypes;
+import org.apache.geode.internal.protocol.protobuf.v1.ListValue;
+import org.apache.geode.internal.protocol.protobuf.v1.Struct;
+import org.apache.geode.internal.protocol.protobuf.v1.Value;
+import org.apache.geode.pdx.PdxInstance;
+import org.apache.geode.test.junit.categories.IntegrationTest;
+
+@RunWith(JUnitQuickcheck.class)
+@Category(IntegrationTest.class)
+public class CompressingProtobufStructSerializerTest {
+
+ private CompressingProtobufStructSerializer serializer;
+ private static Cache cache;
+
+ @BeforeClass
+ public static void createCache() {
+ cache = new CacheFactory().set(ConfigurationProperties.LOG_LEVEL, "error")
+ .setPdxReadSerialized(true).create();
+ }
+
+ @Before
+ public void createSerializer() {
+ serializer = new CompressingProtobufStructSerializer();
+ serializer.init(cache);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cache.close();
+ }
+
+
+ @Property(trials = 10)
+ public void testSymmetry(
+ @PdxInstanceGenerator.ClassName(ProtobufStructSerializer.PROTOBUF_STRUCT) @PdxInstanceGenerator.FieldTypes({
+ String.class, int.class, long.class, byte.class, byte[].class, double.class,
+ PdxInstance.class,
+ ArrayList.class}) @From(PdxInstanceGenerator.class) PdxInstance original)
+ throws IOException, ClassNotFoundException {
+ ByteString bytes = serializer.serialize(original);
+ PdxInstance actual = (PdxInstance) serializer.deserialize(bytes);
+ assertThat(original).isEqualTo(actual);
+ assertEquals(actual, original);
+ }
+}
diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java
index da66f16..13192fc 100644
--- a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java
+++ b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java
@@ -161,15 +161,12 @@ public class ProtobufStructSerializerTest {
@Property(trials = 10)
public void testSymmetry(
- @When(
- seed = 793351614853016898L) @PdxInstanceGenerator.ClassName(ProtobufStructSerializer.PROTOBUF_STRUCT) @PdxInstanceGenerator.FieldTypes({
- String.class, int.class, long.class, byte.class, byte[].class, double.class,
- PdxInstance.class,
- ArrayList.class}) @From(PdxInstanceGenerator.class) PdxInstance original)
+ @PdxInstanceGenerator.ClassName(ProtobufStructSerializer.PROTOBUF_STRUCT) @PdxInstanceGenerator.FieldTypes({
+ String.class, int.class, long.class, byte.class, byte[].class, double.class,
+ PdxInstance.class,
+ ArrayList.class}) @From(PdxInstanceGenerator.class) PdxInstance original)
throws IOException, ClassNotFoundException {
ByteString bytes = serializer.serialize(original);
- Struct struct = Struct.parseFrom(bytes);
- bytes = struct.toByteString();
PdxInstance actual = (PdxInstance) serializer.deserialize(bytes);
assertThat(original).isEqualTo(actual);
assertEquals(actual, original);
--
To stop receiving notification emails like this one, please contact
upthewaterspout@apache.org.