You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by up...@apache.org on 2018/04/11 00:15:43 UTC
[geode] branch feature/transcoding_experiments updated: Handle
nested objects/ArrayLists in ProtobufStructSerializer
This is an automated email from the ASF dual-hosted git repository.
upthewaterspout pushed a commit to branch feature/transcoding_experiments
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/feature/transcoding_experiments by this push:
new 1c7d137 Handle nested objects/ArrayLists in ProtobufStructSerializer
1c7d137 is described below
commit 1c7d1376b0b20cd4169157923bfb12b0994f427e
Author: Dan Smith <up...@apache.org>
AuthorDate: Tue Apr 10 17:13:07 2018 -0700
Handle nested objects/ArrayLists in ProtobufStructSerializer
Adding support for nested objects and lists. Currently, only
PdxInstances with nested PdxInstance or ArrayList fields are supported,
other object types are not supported.
---
.../serialization/ProtobufStructSerializer.java | 150 +++++++++++++--------
.../serialization/PdxInstanceFactoryMock.java | 35 -----
.../serialization/PdxInstanceGenerator.java | 62 +++++++--
.../ProtobufStructSerializerTest.java | 104 +++++++++++---
4 files changed, 232 insertions(+), 119 deletions(-)
diff --git a/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java
index c9a0ba7..d653936 100644
--- a/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java
+++ b/geode-protobuf/src/main/java/org/apache/geode/protocol/serialization/ProtobufStructSerializer.java
@@ -15,13 +15,18 @@
package org.apache.geode.protocol.serialization;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import com.google.protobuf.ByteString;
+import com.google.protobuf.NullValue;
import com.google.protobuf.UnsafeByteOperations;
import org.apache.geode.cache.Cache;
import org.apache.geode.internal.protocol.protobuf.v1.BasicTypes;
+import org.apache.geode.internal.protocol.protobuf.v1.ListValue;
+import org.apache.geode.internal.protocol.protobuf.v1.ProtobufSerializationService;
import org.apache.geode.internal.protocol.protobuf.v1.Struct;
import org.apache.geode.internal.protocol.protobuf.v1.Value;
import org.apache.geode.pdx.PdxInstance;
@@ -33,85 +38,114 @@ public class ProtobufStructSerializer implements ValueSerializer {
@Override
public ByteString serialize(Object object) throws IOException {
+ return serializeStruct(object).toByteString();
+ }
+
+ private Struct serializeStruct(Object object) {
+
PdxInstance pdxInstance = (PdxInstance) object;
Struct.Builder structBuilder = Struct.newBuilder();
for (String fieldName : pdxInstance.getFieldNames()) {
Object value = pdxInstance.getField(fieldName);
- Value.Builder builder = Value.newBuilder();
+ Value serialized = serializeValue(value);
+ structBuilder.putFields(fieldName, serialized);
+ }
+
+ return structBuilder.build();
+ }
+
+ private Value serializeValue(Object value) {
+ Value.Builder builder = Value.newBuilder();
+ if (value instanceof String) {
+ builder.setEncodedValue(
+ BasicTypes.EncodedValue.newBuilder().setStringResult((String) value).build());
+ } else if (value instanceof Boolean) {
+ builder.setEncodedValue(
+ BasicTypes.EncodedValue.newBuilder().setBooleanResult((Boolean) value).build());
+ } else if (value instanceof Integer) {
+ builder.setEncodedValue(
+ BasicTypes.EncodedValue.newBuilder().setIntResult((Integer) value).build());
+ } else if (value instanceof Byte) {
+ builder.setEncodedValue(
+ BasicTypes.EncodedValue.newBuilder().setByteResult((Byte) value).build());
+ } else if (value instanceof Long) {
+ builder.setEncodedValue(
+ BasicTypes.EncodedValue.newBuilder().setLongResult((Long) value).build());
+ } else if (value instanceof byte[]) {
+ builder.setEncodedValue(BasicTypes.EncodedValue.newBuilder()
+ .setBinaryResult(UnsafeByteOperations.unsafeWrap((byte[]) value)).build());
+ } else if (value instanceof PdxInstance) {
+ builder.setStructValue(serializeStruct(value));
+ } else if (value instanceof List) {
+ List<Value> values =
+ ((List<Object>) value).stream().map(this::serializeValue).collect(Collectors.toList());
+ builder.setListValue(ListValue.newBuilder().addAllValues(values).build());
+ } else if (value == null) {
+ builder.setEncodedValue(
+ BasicTypes.EncodedValue.newBuilder().setNullResult(NullValue.NULL_VALUE).build());
+ } else {
+ throw new IllegalStateException(
+ "Don't know how to translate object of type " + value.getClass() + ": " + value);
+ }
+ return builder.build();
+ }
+
+ @Override
+ public Object deserialize(ByteString bytes) throws IOException, ClassNotFoundException {
+ Struct struct = Struct.parseFrom(bytes);
+ return deserialize(struct);
+ }
+
+ private Object deserialize(Struct struct) {
+ PdxInstanceFactory pdxInstanceFactory = cache.createPdxInstanceFactory(PROTOBUF_STRUCT);
+
+ for (Map.Entry<String, Value> field : struct.getFieldsMap().entrySet()) {
+ String fieldName = field.getKey();
+ Object value = deserialize(field.getValue());
if (value instanceof String) {
- builder.setEncodedValue(
- BasicTypes.EncodedValue.newBuilder().setStringResult((String) value).build());
+ pdxInstanceFactory.writeString(fieldName, (String) value);
} else if (value instanceof Boolean) {
- builder.setEncodedValue(
- BasicTypes.EncodedValue.newBuilder().setBooleanResult((Boolean) value).build());
+ pdxInstanceFactory.writeBoolean(fieldName, (Boolean) value);
} else if (value instanceof Integer) {
- builder.setEncodedValue(
- BasicTypes.EncodedValue.newBuilder().setIntResult((Integer) value).build());
+ pdxInstanceFactory.writeInt(fieldName, (Integer) value);
} else if (value instanceof Byte) {
- builder.setEncodedValue(
- BasicTypes.EncodedValue.newBuilder().setByteResult((Byte) value).build());
+ pdxInstanceFactory.writeByte(fieldName, (Byte) value);
} else if (value instanceof Long) {
- builder.setEncodedValue(
- BasicTypes.EncodedValue.newBuilder().setLongResult((Long) value).build());
+ pdxInstanceFactory.writeLong(fieldName, (Long) value);
} else if (value instanceof byte[]) {
- builder.setEncodedValue(BasicTypes.EncodedValue.newBuilder()
- .setBinaryResult(UnsafeByteOperations.unsafeWrap((byte[]) value)).build());
+ pdxInstanceFactory.writeByteArray(fieldName, (byte[]) value);
+ } else if (value instanceof PdxInstance) {
+ pdxInstanceFactory.writeObject(fieldName, value);
+ } else if (value instanceof List) {
+ pdxInstanceFactory.writeObject(fieldName, value);
+ } else if (value == null) {
+ pdxInstanceFactory.writeObject(fieldName, null);
} else {
throw new IllegalStateException(
"Don't know how to translate object of type " + value.getClass() + ": " + value);
}
- structBuilder.putFields(fieldName, builder.build());
}
- return structBuilder.build().toByteString();
+ return pdxInstanceFactory.create();
}
- @Override
- public Object deserialize(ByteString bytes) throws IOException, ClassNotFoundException {
- Struct struct = Struct.parseFrom(bytes);
- PdxInstanceFactory pdxInstanceFactory = cache.createPdxInstanceFactory(PROTOBUF_STRUCT);
+ private List<?> deserialize(List<Value> listValue) {
+ return listValue.stream().map(this::deserialize).collect(Collectors.toList());
+ }
- for (Map.Entry<String, Value> field : struct.getFieldsMap().entrySet()) {
- String fieldName = field.getKey();
- Value value = field.getValue();
- switch (value.getKindCase()) {
- case ENCODEDVALUE:
- switch (value.getEncodedValue().getValueCase()) {
- case STRINGRESULT:
- pdxInstanceFactory.writeString(fieldName, value.getEncodedValue().getStringResult());
- break;
- case BOOLEANRESULT:
- pdxInstanceFactory.writeBoolean(fieldName,
- value.getEncodedValue().getBooleanResult());
- break;
- case INTRESULT:
- pdxInstanceFactory.writeInt(fieldName, value.getEncodedValue().getIntResult());
- break;
- case BYTERESULT:
- pdxInstanceFactory.writeByte(fieldName,
- (byte) value.getEncodedValue().getByteResult());
- break;
- case LONGRESULT:
- pdxInstanceFactory.writeLong(fieldName, value.getEncodedValue().getLongResult());
- break;
- case BINARYRESULT:
- pdxInstanceFactory.writeByteArray(fieldName,
- value.getEncodedValue().getBinaryResult().toByteArray());
- break;
- default:
- throw new IllegalStateException("Don't know how to translate object of type "
- + value.getEncodedValue().getValueCase() + ": " + value);
- }
- break;
-
- default:
- throw new IllegalStateException(
- "Don't know how to translate object of type " + value.getKindCase() + ": " + value);
- }
+ private Object deserialize(Value value) {
+ switch (value.getKindCase()) {
+ case ENCODEDVALUE:
+ return new ProtobufSerializationService().decode(value.getEncodedValue());
+ case STRUCTVALUE:
+ return deserialize(value.getStructValue());
+ case LISTVALUE:
+ return deserialize(value.getListValue().getValuesList());
+ default:
+ throw new IllegalStateException(
+ "Don't know how to translate object of type " + value.getKindCase() + ": " + value);
}
-
- return pdxInstanceFactory.create();
}
@Override
diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceFactoryMock.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceFactoryMock.java
deleted file mode 100644
index 32d477e..0000000
--- a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceFactoryMock.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.geode.protocol.serialization;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import org.apache.geode.internal.cache.InternalCache;
-import org.apache.geode.pdx.PdxInstanceFactory;
-import org.apache.geode.pdx.internal.PdxInstanceFactoryImpl;
-import org.apache.geode.pdx.internal.TypeRegistration;
-import org.apache.geode.pdx.internal.TypeRegistry;
-
-public class PdxInstanceFactoryMock {
-
- public static final PdxInstanceFactory createMockFactory(String className) {
- TypeRegistration registration = mock(TypeRegistration.class);
- InternalCache cache = mock(InternalCache.class);
- when(cache.getPdxRegistry()).thenReturn(new TypeRegistry(registration));
-
- return PdxInstanceFactoryImpl.newCreator(ProtobufStructSerializer.PROTOBUF_STRUCT, true, cache);
- }
-}
diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java
index 0985e7e..5f13f6f 100644
--- a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java
+++ b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/PdxInstanceGenerator.java
@@ -24,9 +24,12 @@ import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -35,12 +38,35 @@ import com.pholser.junit.quickcheck.generator.Generator;
import com.pholser.junit.quickcheck.generator.GeneratorConfiguration;
import com.pholser.junit.quickcheck.random.SourceOfRandomness;
+import org.apache.geode.cache.CacheFactory;
import org.apache.geode.pdx.PdxInstance;
import org.apache.geode.pdx.PdxInstanceFactory;
public class PdxInstanceGenerator extends Generator {
+ private static final Map<Class<?>, Method> supportedTypes;
+ public static final GenerationStatus.Key DEPTH = new GenerationStatus.Key("depth", Integer.class);
+
+ static {
+ HashMap<Class<?>, Method> types = new HashMap<>();
+ Method[] methods = PdxInstanceFactory.class.getDeclaredMethods();
+ for (Method method : methods) {
+ if (method.getName().startsWith("write") && method.getParameterTypes().length == 2) {
+ Class<?> type = method.getParameterTypes()[1];
+
+ if (type == Object.class) {
+ types.put(PdxInstance.class, method);
+ types.put(ArrayList.class, method);
+ } else {
+ types.put(type, method);
+ }
+ }
+ }
+
+ supportedTypes = Collections.unmodifiableMap(types);
+
+ }
private Set<Class<?>> allowedFieldTypes;
private String className = "NO_CLASS";
@@ -55,14 +81,31 @@ public class PdxInstanceGenerator extends Generator {
Map<Class<?>, Method> writeMethods = getAllowedWriteMethods();
int numFields = random.nextInt(0, 20);
- PdxInstanceFactory factory = PdxInstanceFactoryMock.createMockFactory(className);
+ PdxInstanceFactory factory = CacheFactory.getAnyInstance().createPdxInstanceFactory(className);
Set<String> fieldNames =
new HashSet<>(gen().type(String.class).times(numFields).generate(random, status));
for (String fieldName : fieldNames) {
Map.Entry<Class<?>, Method> writeMethod = random.choose(writeMethods.entrySet());
Class<?> type = writeMethod.getKey();
Method method = writeMethod.getValue();
- Object value = gen().type(type).generate(random, status);
+ Object value = null;
+ if (type == PdxInstance.class) {
+ int depth = (int) status.valueOf(DEPTH).orElse(0);
+ if (depth < status.size()) {
+ status.setValue(DEPTH, depth + 1);
+ value = generate(random, status);
+ }
+ } else if (type == ArrayList.class) {
+ int depth = (int) status.valueOf(DEPTH).orElse(0);
+ if (depth < status.size()) {
+ status.setValue(DEPTH, depth + 1);
+ ArrayList<PdxInstance> list = new ArrayList<>();
+ list.add((PdxInstance) generate(random, status));
+ value = list;
+ }
+ } else {
+ value = gen().type(type).generate(random, status);
+ }
try {
method.invoke(factory, fieldName, value);
} catch (IllegalAccessException | InvocationTargetException e) {
@@ -74,16 +117,13 @@ public class PdxInstanceGenerator extends Generator {
}
private Map<Class<?>, Method> getAllowedWriteMethods() {
- final Map<Class<?>, Method> writeMethods = new HashMap<>();
+ final Map<Class<?>, Method> writeMethods = new HashMap<>(supportedTypes);
+ writeMethods.keySet().retainAll(allowedFieldTypes);
- Method[] methods = PdxInstanceFactory.class.getDeclaredMethods();
- for (Method method : methods) {
- if (method.getName().startsWith("write") && method.getParameterTypes().length == 2
- && (allowedFieldTypes == null
- || allowedFieldTypes.contains(method.getParameterTypes()[1]))) {
- Class<?> type = method.getParameterTypes()[1];
- writeMethods.put(type, method);
- }
+ if (writeMethods.size() != allowedFieldTypes.size()) {
+ HashSet<Class<?>> classes = new HashSet<>(allowedFieldTypes);
+ classes.removeAll(supportedTypes.keySet());
+ throw new IllegalStateException("Cannot generate value of types " + classes);
}
return writeMethods;
diff --git a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java
index 294f235..2f1401e 100644
--- a/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java
+++ b/geode-protobuf/src/test/java/org/apache/geode/protocol/serialization/ProtobufStructSerializerTest.java
@@ -21,6 +21,9 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import com.google.protobuf.ByteString;
import com.pholser.junit.quickcheck.From;
@@ -28,14 +31,18 @@ import com.pholser.junit.quickcheck.Property;
import com.pholser.junit.quickcheck.When;
import com.pholser.junit.quickcheck.runner.JUnitQuickcheck;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.apache.geode.cache.Cache;
import org.apache.geode.cache.CacheFactory;
+import org.apache.geode.distributed.ConfigurationProperties;
import org.apache.geode.internal.protocol.protobuf.v1.BasicTypes;
+import org.apache.geode.internal.protocol.protobuf.v1.ListValue;
import org.apache.geode.internal.protocol.protobuf.v1.Struct;
import org.apache.geode.internal.protocol.protobuf.v1.Value;
import org.apache.geode.pdx.PdxInstance;
@@ -47,50 +54,117 @@ import org.apache.geode.test.junit.categories.UnitTest;
public class ProtobufStructSerializerTest {
private ProtobufStructSerializer serializer;
- private Cache cache;
+ private static Cache cache;
+
+ @BeforeClass
+ public static void createCache() {
+ cache = new CacheFactory().set(ConfigurationProperties.LOG_LEVEL, "error")
+ .setPdxReadSerialized(true).create();
+ }
@Before
public void createSerializer() {
- cache = mock(Cache.class);
- when(cache.createPdxInstanceFactory(any()))
- .then(invocation -> PdxInstanceFactoryMock.createMockFactory(invocation.getArgument(0)));
serializer = new ProtobufStructSerializer();
serializer.init(cache);
}
- @After
- public void tearDown() {
+ @AfterClass
+ public static void tearDown() {
cache.close();
}
@Test
public void testDeserialize() throws IOException, ClassNotFoundException {
- Struct struct = Struct.newBuilder()
- .putFields("field1", Value.newBuilder()
- .setEncodedValue(BasicTypes.EncodedValue.newBuilder().setStringResult("value")).build())
- .build();
+ Struct struct = structWithStringField();
ByteString bytes = struct.toByteString();
PdxInstance value = (PdxInstance) serializer.deserialize(bytes);
assertEquals("value", value.getField("field1"));
}
+ private Struct structWithStringField() {
+ return Struct.newBuilder()
+ .putFields("field1", Value.newBuilder()
+ .setEncodedValue(BasicTypes.EncodedValue.newBuilder().setStringResult("value")).build())
+ .build();
+ }
+
@Test
public void testSerialize() throws IOException, ClassNotFoundException {
- PdxInstance value = cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT)
- .writeString("field1", "value").create();
+ PdxInstance value = pdxWithStringField();
ByteString bytes = serializer.serialize(value);
Struct struct = Struct.parseFrom(bytes);
assertEquals("value", struct.getFieldsMap().get("field1").getEncodedValue().getStringResult());
}
- @Property(trials = 100)
+ private PdxInstance pdxWithStringField() {
+ return cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT)
+ .writeString("field1", "value").create();
+ }
+
+ @Test
+ public void canSerializeWithNestedPdxInstance() throws IOException, ClassNotFoundException {
+ PdxInstance value = cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT)
+ .writeObject("field1", pdxWithStringField()).create();
+ ByteString bytes = serializer.serialize(value);
+ Struct struct = Struct.parseFrom(bytes);
+
+ assertEquals("value", struct.getFieldsMap().get("field1").getStructValue().getFieldsMap()
+ .get("field1").getEncodedValue().getStringResult());
+ }
+
+ @Test
+ public void canSerializeWithNestedList() throws IOException, ClassNotFoundException {
+ ArrayList<PdxInstance> list = new ArrayList<>();
+ list.add(pdxWithStringField());
+ PdxInstance value = cache.createPdxInstanceFactory(ProtobufStructSerializer.PROTOBUF_STRUCT)
+ .writeObject("field2", list).create();
+ ByteString bytes = serializer.serialize(value);
+ Struct struct = Struct.parseFrom(bytes);
+
+ assertEquals(Struct.newBuilder()
+ .putFields("field2",
+ Value.newBuilder()
+ .setListValue(ListValue.newBuilder()
+ .addValues(Value.newBuilder().setStructValue(structWithStringField())))
+ .build())
+ .build(), struct);
+ }
+
+ @Test
+ public void canDeserializeWithNestedStruct() throws IOException, ClassNotFoundException {
+ Struct.Builder builder = Struct.newBuilder();
+ builder.putFields("field1", Value.newBuilder().setStructValue(structWithStringField()).build());
+ ByteString bytes = builder.build().toByteString();
+ PdxInstance value = (PdxInstance) serializer.deserialize(bytes);
+
+ PdxInstance nested = (PdxInstance) value.getField("field1");
+ assertEquals("value", nested.getField("field1"));
+ }
+
+ @Test
+ public void canDeserializeWithNestedList() throws IOException, ClassNotFoundException {
+ Struct.Builder builder = Struct.newBuilder();
+ builder.putFields("field1",
+ Value.newBuilder()
+ .setListValue(ListValue.newBuilder().addValues(Value.newBuilder()
+ .setEncodedValue(BasicTypes.EncodedValue.newBuilder().setStringResult("value"))))
+ .build());
+ ByteString bytes = builder.build().toByteString();
+ PdxInstance value = (PdxInstance) serializer.deserialize(bytes);
+
+ List<String> nested = (List<String>) value.getField("field1");
+ assertEquals(Arrays.asList("value"), nested);
+ }
+
+
+ @Property(trials = 10)
public void testSymmetry(
@When(
seed = 793351614853016898L) @PdxInstanceGenerator.ClassName(ProtobufStructSerializer.PROTOBUF_STRUCT) @PdxInstanceGenerator.FieldTypes({
- String.class, int.class, long.class, byte.class,
- byte[].class}) @From(PdxInstanceGenerator.class) PdxInstance original)
+ String.class, int.class, long.class, byte.class, byte[].class, PdxInstance.class,
+ ArrayList.class}) @From(PdxInstanceGenerator.class) PdxInstance original)
throws IOException, ClassNotFoundException {
ByteString bytes = serializer.serialize(original);
Struct struct = Struct.parseFrom(bytes);
--
To stop receiving notification emails like this one, please contact
upthewaterspout@apache.org.