You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2019/09/30 03:43:05 UTC
[camel] branch master updated: CAMEL-13988: Add Map data input
support to Protobuf dataformat (#3206)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/master by this push:
new 96fc480 CAMEL-13988: Add Map data input support to Protobuf dataformat (#3206)
96fc480 is described below
commit 96fc4807e0f4c46ff846cc0610e7e7aa1920276d
Author: Omar Al-Safi <om...@gmail.com>
AuthorDate: Mon Sep 30 05:42:59 2019 +0200
CAMEL-13988: Add Map data input support to Protobuf dataformat (#3206)
* CAMEL-13988: Initial protobuf converter from Map to Proto Message
Signed-off-by: Omar Al-Safi <om...@gmail.com>
* CAMEL-13988: Add tests and modify documentation for protobuf dataformat
Signed-off-by: Omar Al-Safi <om...@gmail.com>
---
.../src/main/docs/protobuf-dataformat.adoc | 8 +-
.../dataformat/protobuf/ProtobufConverter.java | 120 +++++++++++++++++++++
.../dataformat/protobuf/ProtobufDataFormat.java | 21 +++-
.../dataformat/protobuf/ProtobufConverterTest.java | 101 +++++++++++++++++
.../ProtobufMarshalAndUnmarshalMapTest.java | 84 +++++++++++++++
.../src/test/proto/addressbook.proto | 9 ++
6 files changed, 337 insertions(+), 6 deletions(-)
diff --git a/components/camel-protobuf/src/main/docs/protobuf-dataformat.adoc b/components/camel-protobuf/src/main/docs/protobuf-dataformat.adoc
index cf5fc9d..3d53806 100644
--- a/components/camel-protobuf/src/main/docs/protobuf-dataformat.adoc
+++ b/components/camel-protobuf/src/main/docs/protobuf-dataformat.adoc
@@ -82,6 +82,10 @@ from("direct:marshal")
.to("mock:reverse");
--------------------------------------------------------------------------------------------------
+== Input data type
+This dataformat supports marshaling input data either as protobuf `Message` type or `Map` data type. In case of input data as `Map` type, first it will try to retrieve the data as `Map` using built-in type converters, if it fails to
+do so, it will fall back to retrieve it as proto `Message`.
+
== Protobuf overview
This quick overview of how to use Protobuf. For more detail see the
@@ -196,11 +200,11 @@ DataFormat marshal and unmarshal API like this.
-----------------------------------------------------------------------------------
Or use the DSL protobuf() passing the unmarshal default instance or
-default instance class name like this.
+default instance class name like this. However, if you have input data as `Map` type, you will need to **specify** the ProtobufDataFormat otherwise it will throw an error.
[source,java]
--------------------------------------------------------------------------------------------------
- // You don't need to specify the default instance for protobuf marshaling
+ // You don't need to specify the default instance for protobuf marshaling, but you will need in case your input data is a Map type
from("direct:marshal").marshal().protobuf();
from("direct:unmarshalA").unmarshal()
.protobuf("org.apache.camel.dataformat.protobuf.generated.AddressBookProtos$Person")
diff --git a/components/camel-protobuf/src/main/java/org/apache/camel/dataformat/protobuf/ProtobufConverter.java b/components/camel-protobuf/src/main/java/org/apache/camel/dataformat/protobuf/ProtobufConverter.java
new file mode 100644
index 0000000..a4d2acf
--- /dev/null
+++ b/components/camel-protobuf/src/main/java/org/apache/camel/dataformat/protobuf/ProtobufConverter.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.protobuf;
+
+import java.util.List;
+import java.util.Map;
+
+import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.EnumValueDescriptor;
+import com.google.protobuf.Descriptors.FieldDescriptor;
+import com.google.protobuf.Message;
+import com.google.protobuf.Message.Builder;
+import org.apache.camel.util.ObjectHelper;
+
+public final class ProtobufConverter {
+
+ private final Message defaultInstance;
+
+ private ProtobufConverter(final Message defaultInstance) {
+ this.defaultInstance = defaultInstance;
+ }
+
+ public static ProtobufConverter create(final Message defaultInstance) {
+ ObjectHelper.notNull(defaultInstance, "defaultInstance");
+
+ return new ProtobufConverter(defaultInstance);
+ }
+
+ public Message toProto(final Map<?, ?> inputData) {
+ ObjectHelper.notNull(inputData, "inputData");
+
+ final Descriptor descriptor = defaultInstance.getDescriptorForType();
+ final Builder target = defaultInstance.newBuilderForType();
+
+ return convertMapToMessage(descriptor, target, inputData);
+ }
+
+ private Message convertMapToMessage(final Descriptor descriptor, final Builder builder, final Map<?, ?> inputData) {
+ ObjectHelper.notNull(descriptor, "descriptor");
+ ObjectHelper.notNull(builder, "builder");
+ ObjectHelper.notNull(inputData, "inputData");
+
+ // we set our values from map to descriptor
+ inputData.forEach((key, value) -> {
+ final FieldDescriptor fieldDescriptor = descriptor.findFieldByName(key.toString());
+ // if we don't find our desired fieldDescriptor, we just ignore it
+ if (fieldDescriptor != null) {
+ if (fieldDescriptor.isRepeated()) {
+ final List<?> repeatedValues = castValue(value, List.class, String.format("Not able to cast value to list, make sure you have a list for the repeated field '%s'", fieldDescriptor.getName()));
+ repeatedValues.forEach(repeatedValue -> builder.addRepeatedField(fieldDescriptor, getSuitableFieldValue(fieldDescriptor, builder, repeatedValue)));
+ } else {
+ builder.setField(fieldDescriptor, getSuitableFieldValue(fieldDescriptor, builder, value));
+ }
+ }
+ });
+ return builder.build();
+ }
+
+ private Object getSuitableFieldValue(final FieldDescriptor fieldDescriptor, final Builder builder, final Object inputValue) {
+ ObjectHelper.notNull(fieldDescriptor, "fieldDescriptor");
+ ObjectHelper.notNull(builder, "builder");
+ ObjectHelper.notNull(inputValue, "inputValue");
+
+ switch (fieldDescriptor.getJavaType()) {
+ case ENUM:
+ return getEnumValue(fieldDescriptor, inputValue);
+
+ case MESSAGE:
+ final Map<?, ?> nestedValue = castValue(inputValue, Map.class, String.format("Not able to cast value to map, make sure you have a map for the nested field message '%s'", fieldDescriptor.getName()));
+ // we do a nested call until we reach our final message
+ return convertMapToMessage(fieldDescriptor.getMessageType(), builder.newBuilderForField(fieldDescriptor), nestedValue);
+
+ default:
+ return inputValue;
+ }
+ }
+
+ private EnumValueDescriptor getEnumValue(final FieldDescriptor fieldDescriptor, final Object value) {
+ final EnumValueDescriptor enumValueDescriptor = getSuitableEnumValue(fieldDescriptor, value);
+
+ if (enumValueDescriptor == null) {
+ throw new IllegalArgumentException(String.format("Could not retrieve enum index '%s' for enum field '%s', most likely the index does not exist in the enum indexes '%s'",
+ value, fieldDescriptor.getName(), fieldDescriptor.getEnumType().getValues()));
+ }
+
+ return enumValueDescriptor;
+ }
+
+ private EnumValueDescriptor getSuitableEnumValue(final FieldDescriptor fieldDescriptor, final Object value) {
+ // we check if value is string, we find index by name, otherwise by integer
+ if (value instanceof String) {
+ return fieldDescriptor.getEnumType().findValueByName((String) value);
+ } else {
+ final int index = castValue(value, Integer.class, String.format("Not able to cast value to integer, make sure you have an integer index for the enum field '%s'", fieldDescriptor.getName()));
+ return fieldDescriptor.getEnumType().findValueByNumber(index);
+ }
+ }
+
+ private static <T> T castValue(final Object value, final Class<T> type, final String errorMessage) {
+ try {
+ return type.cast(value);
+ } catch (ClassCastException e) {
+ throw new IllegalArgumentException(errorMessage, e);
+ }
+ }
+}
diff --git a/components/camel-protobuf/src/main/java/org/apache/camel/dataformat/protobuf/ProtobufDataFormat.java b/components/camel-protobuf/src/main/java/org/apache/camel/dataformat/protobuf/ProtobufDataFormat.java
index c8f2f3c..b14b28e 100644
--- a/components/camel-protobuf/src/main/java/org/apache/camel/dataformat/protobuf/ProtobufDataFormat.java
+++ b/components/camel-protobuf/src/main/java/org/apache/camel/dataformat/protobuf/ProtobufDataFormat.java
@@ -20,6 +20,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.lang.reflect.Method;
+import java.util.Map;
import com.google.protobuf.Message;
import com.google.protobuf.Message.Builder;
@@ -29,6 +30,7 @@ import org.apache.camel.CamelContextAware;
import org.apache.camel.CamelException;
import org.apache.camel.Exchange;
import org.apache.camel.InvalidPayloadException;
+import org.apache.camel.NoTypeConversionAvailableException;
import org.apache.camel.spi.DataFormat;
import org.apache.camel.spi.DataFormatName;
import org.apache.camel.spi.annotations.Dataformat;
@@ -121,25 +123,36 @@ public class ProtobufDataFormat extends ServiceSupport implements DataFormat, Da
*/
@Override
public void marshal(final Exchange exchange, final Object graph, final OutputStream outputStream) throws Exception {
+ final Message inputMessage = convertGraphToMessage(exchange, graph);
+
String contentTypeHeader = CONTENT_TYPE_HEADER_NATIVE;
if (contentTypeFormat.equals(CONTENT_TYPE_FORMAT_JSON)) {
- IOUtils.write(JsonFormat.printer().print((Message)graph), outputStream, "UTF-8");
+ IOUtils.write(JsonFormat.printer().print(inputMessage), outputStream, "UTF-8");
contentTypeHeader = CONTENT_TYPE_HEADER_JSON;
} else if (contentTypeFormat.equals(CONTENT_TYPE_FORMAT_NATIVE)) {
- ((Message)graph).writeTo(outputStream);
+ inputMessage.writeTo(outputStream);
} else {
throw new CamelException("Invalid protobuf content type format: " + contentTypeFormat);
}
if (isContentTypeHeader()) {
- if (exchange.hasOut()) {
- exchange.getOut().setHeader(Exchange.CONTENT_TYPE, contentTypeHeader);
+ if (exchange.getMessage() != null) {
+ exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, contentTypeHeader);
} else {
exchange.getIn().setHeader(Exchange.CONTENT_TYPE, contentTypeHeader);
}
}
}
+ private Message convertGraphToMessage(final Exchange exchange, final Object inputData) throws NoTypeConversionAvailableException {
+ final Map<?, ?> messageInMap = exchange.getContext().getTypeConverter().tryConvertTo(Map.class, exchange, inputData);
+ if (messageInMap != null) {
+ final ProtobufConverter protobufConverter = ProtobufConverter.create(defaultInstance);
+ return protobufConverter.toProto(messageInMap);
+ }
+ return exchange.getContext().getTypeConverter().mandatoryConvertTo(Message.class, exchange, inputData);
+ }
+
/*
* (non-Javadoc)
* @see org.apache.camel.spi.DataFormat#unmarshal(org.apache.camel.Exchange,
diff --git a/components/camel-protobuf/src/test/java/org/apache/camel/dataformat/protobuf/ProtobufConverterTest.java b/components/camel-protobuf/src/test/java/org/apache/camel/dataformat/protobuf/ProtobufConverterTest.java
new file mode 100644
index 0000000..b7f22e4
--- /dev/null
+++ b/components/camel-protobuf/src/test/java/org/apache/camel/dataformat/protobuf/ProtobufConverterTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.protobuf;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.camel.dataformat.protobuf.generated.AddressBookProtos;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ProtobufConverterTest {
+
+ @Test
+ public void testIfCorrectlyParseMap() {
+ final Map<String, Object> phoneNumber = new HashMap<>();
+ phoneNumber.put("number", "011122233");
+ phoneNumber.put("type", "MOBILE");
+
+ final Map<String, Object> phoneNumber2 = new HashMap<>();
+ phoneNumber2.put("number", "5542454");
+ phoneNumber2.put("type", 2);
+
+ final Map<String, Object> address = new HashMap<>();
+ address.put("street", "awesome street");
+ address.put("street_number", 12);
+ address.put("is_valid", false);
+
+ final Map<String, Object> input = new HashMap<>();
+
+ input.put("name", "Martin");
+ input.put("id", 1234);
+ input.put("phone", Arrays.asList(phoneNumber, phoneNumber2));
+ input.put("email", "some@some.com");
+ input.put("nicknames", Arrays.asList("awesome1", "awesome2"));
+ input.put("address", address);
+
+ final ProtobufConverter protobufConverter = ProtobufConverter.create(AddressBookProtos.Person.getDefaultInstance());
+ final AddressBookProtos.Person message = (AddressBookProtos.Person) protobufConverter.toProto(input);
+
+ // assert primitives types and strings
+ assertEquals("Martin", message.getName());
+ assertEquals(1234, message.getId());
+ assertEquals("some@some.com", message.getEmail());
+
+ // assert nested message
+ assertEquals("awesome street", message.getAddress().getStreet());
+ assertEquals(12, message.getAddress().getStreetNumber());
+ assertFalse(message.getAddress().getIsValid());
+
+ // assert repeated messages
+ assertEquals("011122233", message.getPhone(0).getNumber());
+ assertEquals("MOBILE", message.getPhone(0).getType().name());
+ assertEquals("5542454", message.getPhone(1).getNumber());
+ assertEquals("WORK", message.getPhone(1).getType().name());
+
+ assertEquals("awesome1", message.getNicknames(0));
+ assertEquals("awesome2", message.getNicknames(1));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIfThrowsErrorInCaseNestedMessageNotMap() {
+ final Map<String, Object> input = new HashMap<>();
+
+ input.put("name", "Martin");
+ input.put("id", 1234);
+ input.put("address", "wrong address");
+
+ final ProtobufConverter protobufConverter = ProtobufConverter.create(AddressBookProtos.Person.getDefaultInstance());
+ final AddressBookProtos.Person message = (AddressBookProtos.Person) protobufConverter.toProto(input);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIfThrowsErrorInCaseRepeatedFieldIsNotList() {
+ final Map<String, Object> input = new HashMap<>();
+
+ input.put("name", "Martin");
+ input.put("id", 1234);
+ input.put("nicknames", "wrong nickname");
+
+ final ProtobufConverter protobufConverter = ProtobufConverter.create(AddressBookProtos.Person.getDefaultInstance());
+ final AddressBookProtos.Person message = (AddressBookProtos.Person) protobufConverter.toProto(input);
+ }
+
+}
\ No newline at end of file
diff --git a/components/camel-protobuf/src/test/java/org/apache/camel/dataformat/protobuf/ProtobufMarshalAndUnmarshalMapTest.java b/components/camel-protobuf/src/test/java/org/apache/camel/dataformat/protobuf/ProtobufMarshalAndUnmarshalMapTest.java
new file mode 100644
index 0000000..a4c4eb3
--- /dev/null
+++ b/components/camel-protobuf/src/test/java/org/apache/camel/dataformat/protobuf/ProtobufMarshalAndUnmarshalMapTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.protobuf;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.dataformat.protobuf.generated.AddressBookProtos.Person;
+import org.apache.camel.test.junit4.CamelTestSupport;
+import org.junit.Test;
+
+public class ProtobufMarshalAndUnmarshalMapTest extends CamelTestSupport {
+
+ @Test
+ public void testMarshalAndUnmarshal() throws Exception {
+ marshalAndUnmarshal("direct:in", "direct:back");
+ }
+
+ @Test
+ public void testMarshalAndUnmarshalWithDSL() throws Exception {
+ marshalAndUnmarshal("direct:marshal", "direct:unmarshalA");
+ }
+
+ private void marshalAndUnmarshal(String inURI, String outURI) throws Exception {
+ final Map<String, Object> input = new HashMap<>();
+ final Map<String, Object> phoneNumber = new HashMap<>();
+ phoneNumber.put("number", "011122233");
+ phoneNumber.put("type", 0);
+
+ input.put("name", "Martin");
+ input.put("id", 1234);
+ input.put("phone", Collections.singletonList(phoneNumber));
+
+ MockEndpoint mock = getMockEndpoint("mock:reverse");
+ mock.expectedMessageCount(1);
+ mock.message(0).body().isInstanceOf(Person.class);
+
+ Object marshalled = template.requestBody(inURI, input);
+
+ template.sendBody(outURI, marshalled);
+
+ mock.assertIsSatisfied();
+
+ Person output = mock.getReceivedExchanges().get(0).getIn().getBody(Person.class);
+ assertEquals("Martin", output.getName());
+ assertEquals(1234, output.getId());
+ assertEquals("011122233", output.getPhone(0).getNumber());
+ assertEquals(0, output.getPhone(0).getType().getNumber());
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ ProtobufDataFormat format = new ProtobufDataFormat(Person.getDefaultInstance());
+
+ from("direct:in").marshal(format);
+ from("direct:back").unmarshal(format).to("mock:reverse");
+
+ from("direct:marshal").marshal().protobuf("org.apache.camel.dataformat.protobuf.generated.AddressBookProtos$Person");
+ from("direct:unmarshalA").unmarshal().protobuf("org.apache.camel.dataformat.protobuf.generated.AddressBookProtos$Person").to("mock:reverse");
+ }
+ };
+ }
+
+}
diff --git a/components/camel-protobuf/src/test/proto/addressbook.proto b/components/camel-protobuf/src/test/proto/addressbook.proto
index 7fce508..e5412a3 100644
--- a/components/camel-protobuf/src/test/proto/addressbook.proto
+++ b/components/camel-protobuf/src/test/proto/addressbook.proto
@@ -22,6 +22,15 @@ message Person {
}
repeated PhoneNumber phone = 4;
+ repeated string nicknames = 5;
+
+ message Address {
+ optional string street = 1;
+ optional int32 street_number = 2;
+ optional bool is_valid = 3;
+ }
+
+ optional Address address = 6;
}
message AddressBook {