[incubator-servicecomb-java-chassis] 04/09: [SCB-918][WIP] create codec mechanism

[li...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

liubao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-servicecomb-java-chassis.git

commit f4960d0275a2571f3b730d88564015c08d61f5ac
Author: wujimin <wu...@huawei.com>
AuthorDate: Thu Sep 27 22:21:55 2018 +0800

    [SCB-918][WIP] create codec mechanism
---
 foundations/foundation-protobuf/pom.xml            |  13 +
 .../java/io/protostuff/runtime/MessageSchema.java  | 268 +++++++++++++++++++++
 .../foundation/protobuf/ProtoMapper.java           | 108 +++++++++
 .../foundation/protobuf/ProtoMapperFactory.java    |  62 +++++
 .../foundation/protobuf/RootDeserializer.java      |  60 +++++
 .../foundation/protobuf/RootSerializer.java        |  52 ++++
 6 files changed, 563 insertions(+)

diff --git a/foundations/foundation-protobuf/pom.xml b/foundations/foundation-protobuf/pom.xml
index 48d92eb..a5e1416 100644
--- a/foundations/foundation-protobuf/pom.xml
+++ b/foundations/foundation-protobuf/pom.xml
@@ -37,6 +37,14 @@
       <artifactId>protostuff-parser</artifactId>
     </dependency>
     <dependency>
+      <groupId>io.protostuff</groupId>
+      <artifactId>protostuff-runtime</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.protostuff</groupId>
+      <artifactId>protostuff-core</artifactId>
+    </dependency>
+    <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
     </dependency>
@@ -47,5 +55,10 @@
       <version>3.6.1</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-protobuf</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>
\ No newline at end of file
diff --git a/foundations/foundation-protobuf/src/main/java/io/protostuff/runtime/MessageSchema.java b/foundations/foundation-protobuf/src/main/java/io/protostuff/runtime/MessageSchema.java
new file mode 100644
index 0000000..bd49572
--- /dev/null
+++ b/foundations/foundation-protobuf/src/main/java/io/protostuff/runtime/MessageSchema.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.protostuff.runtime;
+
+import static io.protostuff.runtime.RuntimeSchema.MIN_TAG_FOR_HASH_FIELD_MAP;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.servicecomb.foundation.common.concurrent.ConcurrentHashMapEx;
+import org.apache.servicecomb.foundation.common.utils.bean.Getter;
+import org.apache.servicecomb.foundation.protobuf.ProtoMapper;
+import org.apache.servicecomb.foundation.protobuf.internal.bean.BeanDescriptor;
+import org.apache.servicecomb.foundation.protobuf.internal.bean.PropertyDescriptor;
+import org.apache.servicecomb.foundation.protobuf.internal.schema.FieldSchema;
+import org.apache.servicecomb.foundation.protobuf.internal.schema.serializer.PojoFieldSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.protostuff.Input;
+import io.protostuff.Output;
+import io.protostuff.Schema;
+import io.protostuff.compiler.model.Message;
+import io.protostuff.runtime.RuntimeEnv.Instantiator;
+
+public class MessageSchema implements Schema<Object>, FieldMap<Object> {
+  private static final Logger LOGGER = LoggerFactory.getLogger(MessageSchema.class);
+
+  private ProtoMapper protoMapper;
+
+  private FieldMap<Object> fieldMap;
+
+  private Class<Object> typeClass;
+
+  private Instantiator<Object> instantiator;
+
+  private Message message;
+
+  // one class can bind to different proto message (almost different version)
+  // so save the information only in message, not global
+  private final Map<Type, List<PojoFieldSerializer>> pojoFieldSerializers = new ConcurrentHashMapEx<>();
+
+  public void init(ProtoMapper protoMapper, Collection<Field<Object>> fields, Message message) {
+    init(protoMapper, null, fields, null, message);
+  }
+
+  public void init(ProtoMapper protoMapper, Class<Object> typeClass, Collection<Field<Object>> fields,
+      Instantiator<Object> instantiator,
+      Message message) {
+    this.protoMapper = protoMapper;
+    this.fieldMap = createFieldMap(fields);
+    this.instantiator = instantiator;
+    this.typeClass = typeClass;
+    this.message = message;
+  }
+
+  public Message getMessage() {
+    return message;
+  }
+
+  private FieldMap<Object> createFieldMap(Collection<Field<Object>> fields) {
+    int lastFieldNumber = 0;
+    for (Field<Object> field : fields) {
+      if (field.number > lastFieldNumber) {
+        lastFieldNumber = field.number;
+      }
+    }
+    if (preferHashFieldMap(fields, lastFieldNumber)) {
+      return new HashFieldMap<>(fields);
+    }
+    // array field map should be more efficient
+    return new ArrayFieldMap<>(fields, lastFieldNumber);
+  }
+
+  private boolean preferHashFieldMap(Collection<Field<Object>> fields, int lastFieldNumber) {
+    return lastFieldNumber > MIN_TAG_FOR_HASH_FIELD_MAP && lastFieldNumber >= 2 * fields.size();
+  }
+
+  @Override
+  public String getFieldName(int number) {
+    // only called on writes
+    final Field<Object> field = fieldMap.getFieldByNumber(number);
+    return field == null ? null : field.name;
+  }
+
+  @Override
+  public int getFieldNumber(String name) {
+    final Field<Object> field = fieldMap.getFieldByName(name);
+    return field == null ? null : field.number;
+  }
+
+  @Override
+  public boolean isInitialized(Object message) {
+    return true;
+  }
+
+  @Override
+  public Object newMessage() {
+    return instantiator.newInstance();
+  }
+
+  @Override
+  public String messageName() {
+    return message.getName();
+  }
+
+  @Override
+  public String messageFullName() {
+    return message.getCanonicalName();
+  }
+
+  @Override
+  public Class<? super Object> typeClass() {
+    return typeClass;
+  }
+
+  @Override
+  public void mergeFrom(Input input, Object message) throws IOException {
+    Field<Object> field = null;
+    try {
+      for (int n = input.readFieldNumber(this); n != 0; n = input.readFieldNumber(this)) {
+        field = fieldMap.getFieldByNumber(n);
+        if (field == null) {
+          input.handleUnknownField(n, this);
+        } else {
+          field.mergeFrom(input, message);
+        }
+      }
+    } catch (IOException e) {
+      logError((FieldSchema) field, "deserialize", e);
+      throw e;
+    } catch (RuntimeException e) {
+      logError((FieldSchema) field, "deserialize", e);
+      throw e;
+    }
+  }
+
+  protected void logError(FieldSchema fieldSchema, String action, Throwable e) {
+    if (fieldSchema == null) {
+      return;
+    }
+
+    io.protostuff.compiler.model.Field protoField = fieldSchema.getProtoField();
+    LOGGER.error("Failed to {}, field={}:{}, type={}",
+        action,
+        protoField.getType().getCanonicalName(),
+        protoField.getName(),
+        protoField.getTypeName(),
+        e.getMessage());
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public void writeTo(Output output, Object message) throws IOException {
+    if (message == null) {
+      return;
+    }
+
+    if (message instanceof Map) {
+      writeFromMap(output, (Map<String, Object>) message);
+      return;
+    }
+
+    writeFromPojo(output, message);
+  }
+
+  /**
+   * <pre>
+   * when use with generic
+   * each time serialize the value field, will run with the real type
+   * so there is no problem
+   *
+   * eg: CustomeGeneric&lt;User&gt; someMethod(CustomGeneric&lt;People&gt; input)
+   *   input: {
+   *     People value
+   *   }
+   *   output: {
+   *     User value
+   *   }
+   * </pre>
+   * @param output
+   * @param value
+   * @throws Throwable
+   */
+  protected void writeFromPojo(Output output, Object value) throws IOException {
+    List<PojoFieldSerializer> serializers = pojoFieldSerializers
+        .computeIfAbsent(value.getClass(), this::createFieldSerializers);
+    for (PojoFieldSerializer serializer : serializers) {
+      serializer.writeTo(output, value);
+    }
+  }
+
+  protected List<PojoFieldSerializer> createFieldSerializers(Type type) {
+    BeanDescriptor beanDescriptor = protoMapper.getBeanDescriptorManager().getOrCreateBeanDescriptor(type);
+    List<PojoFieldSerializer> pojoFieldSerializers = new ArrayList<>();
+    for (Field<Object> f : fieldMap.getFields()) {
+      PropertyDescriptor propertyDescriptor = beanDescriptor.getPropertyDescriptors().get(f.name);
+      if (propertyDescriptor == null) {
+        continue;
+      }
+
+      Getter getter = propertyDescriptor.getGetter();
+      if (getter == null) {
+        continue;
+      }
+
+      pojoFieldSerializers.add(new PojoFieldSerializer(getter, (FieldSchema) f));
+    }
+
+    return pojoFieldSerializers;
+  }
+
+  protected void writeFromMap(Output output, Map<String, Object> map) throws IOException {
+    for (Entry<String, Object> entry : map.entrySet()) {
+      if (entry.getValue() == null) {
+        continue;
+      }
+
+      Field<Object> field = fieldMap.getFieldByName(entry.getKey());
+      if (field == null) {
+        // not defined in proto, ignore it.
+        continue;
+      }
+
+      field.writeTo(output, entry.getValue());
+    }
+  }
+
+  @Override
+  public Field<Object> getFieldByNumber(int n) {
+    return fieldMap.getFieldByNumber(n);
+  }
+
+  @Override
+  public Field<Object> getFieldByName(String fieldName) {
+    return fieldMap.getFieldByName(fieldName);
+  }
+
+  @Override
+  public int getFieldCount() {
+    return fieldMap.getFieldCount();
+  }
+
+  @Override
+  public List<Field<Object>> getFields() {
+    return fieldMap.getFields();
+  }
+}
diff --git a/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/ProtoMapper.java b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/ProtoMapper.java
new file mode 100644
index 0000000..860a9f6
--- /dev/null
+++ b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/ProtoMapper.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.foundation.protobuf;
+
+import java.lang.reflect.Type;
+import java.util.Map;
+
+import org.apache.servicecomb.foundation.common.concurrent.ConcurrentHashMapEx;
+import org.apache.servicecomb.foundation.protobuf.internal.bean.BeanDescriptorManager;
+import org.apache.servicecomb.foundation.protobuf.internal.schema.deserializer.DeserializerSchemaManager;
+import org.apache.servicecomb.foundation.protobuf.internal.schema.serializer.SerializerSchemaManager;
+
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.TypeFactory;
+
+import io.protostuff.compiler.model.Message;
+import io.protostuff.compiler.model.Proto;
+
+public class ProtoMapper {
+  private final ObjectMapper jsonMapper;
+
+  private final BeanDescriptorManager beanDescriptorManager;
+
+  private final Proto proto;
+
+  private final SerializerSchemaManager serializerSchemaManager;
+
+  private final DeserializerSchemaManager deserializerSchemaManager;
+
+  // key is message canonical name
+  // this allowed developer to control any type deserializer
+  // otherwise any type will be deserialized to LinkedHashMap
+  private final Map<String, JavaType> anyTypes = new ConcurrentHashMapEx<>();
+
+  protected ProtoMapper(ObjectMapper jsonMapper, BeanDescriptorManager beanDescriptorManager, Proto proto) {
+    this.jsonMapper = jsonMapper;
+    this.beanDescriptorManager = beanDescriptorManager;
+    this.proto = proto;
+
+    serializerSchemaManager = new SerializerSchemaManager(this);
+    deserializerSchemaManager = new DeserializerSchemaManager(this);
+  }
+
+  public Proto getProto() {
+    return proto;
+  }
+
+  public ObjectMapper getJsonMapper() {
+    return jsonMapper;
+  }
+
+  public BeanDescriptorManager getBeanDescriptorManager() {
+    return beanDescriptorManager;
+  }
+
+  public Map<String, JavaType> getAnyTypes() {
+    return anyTypes;
+  }
+
+  public void addAnyType(String canonicalName, Type type) {
+    anyTypes.put(canonicalName, TypeFactory.defaultInstance().constructType(type));
+  }
+
+  public Message getMessageFromCanonicaName(String messageCanonicalName) {
+    for (Message message : proto.getMessages()) {
+      if (message.getCanonicalName().equals(messageCanonicalName)) {
+        return message;
+      }
+    }
+
+    return null;
+  }
+
+  public RootSerializer findRootSerializer(String shortMessageName) {
+    return serializerSchemaManager.findRootSerializer(shortMessageName);
+  }
+
+  public RootSerializer findRootSerializerByCanonical(String canonicalMessageName) {
+    return serializerSchemaManager.findRootSerializerByCanonical(canonicalMessageName);
+  }
+
+  public RootDeserializer createRootDeserializer(Type type, String shortMessageName) {
+    return createRootDeserializer(TypeFactory.defaultInstance().constructType(type), shortMessageName);
+  }
+
+  public RootDeserializer createRootDeserializer(JavaType javaType, String shortMessageName) {
+    return deserializerSchemaManager.createRootDeserializer(javaType, shortMessageName);
+  }
+
+  public RootDeserializer createRootDeserializer(JavaType javaType, Message message) {
+    return deserializerSchemaManager.createRootDeserializer(javaType, message);
+  }
+}
diff --git a/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/ProtoMapperFactory.java b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/ProtoMapperFactory.java
new file mode 100644
index 0000000..1e1caba
--- /dev/null
+++ b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/ProtoMapperFactory.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.foundation.protobuf;
+
+import org.apache.servicecomb.foundation.protobuf.internal.bean.BeanDescriptorManager;
+import org.apache.servicecomb.foundation.protobuf.internal.parser.ProtoParser;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.protostuff.compiler.model.Proto;
+
+public class ProtoMapperFactory {
+  // 1.to support "any" type
+  // 2.to find bean properties
+  private ObjectMapper jsonMapper = new ObjectMapper();
+
+  private BeanDescriptorManager beanDescriptorManager;
+
+  private ProtoParser protoParser = new ProtoParser();
+
+  public ProtoMapperFactory() {
+    jsonMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+    jsonMapper.setSerializationInclusion(Include.NON_NULL);
+//    jsonMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.OBJECT_AND_NON_CONCRETE, JsonTypeInfo.As.PROPERTY);
+
+    beanDescriptorManager = new BeanDescriptorManager(jsonMapper.getSerializationConfig());
+  }
+
+  public BeanDescriptorManager getBeanDescriptorManager() {
+    return beanDescriptorManager;
+  }
+
+  public ProtoMapper createFromContent(String content) {
+    Proto proto = protoParser.parseFromContent(content);
+    return create(proto);
+  }
+
+  public ProtoMapper createFromName(String name) {
+    Proto proto = protoParser.parse(name);
+    return create(proto);
+  }
+
+  public ProtoMapper create(Proto proto) {
+    return new ProtoMapper(jsonMapper, beanDescriptorManager, proto);
+  }
+}
diff --git a/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/RootDeserializer.java b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/RootDeserializer.java
new file mode 100644
index 0000000..7b600bc
--- /dev/null
+++ b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/RootDeserializer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.foundation.protobuf;
+
+import java.io.IOException;
+
+import org.apache.servicecomb.foundation.protobuf.internal.bean.BeanDescriptor;
+
+import io.protostuff.ByteArrayInput;
+import io.protostuff.Input;
+import io.protostuff.runtime.MessageSchema;
+
+public class RootDeserializer {
+  private BeanDescriptor beanDescriptor;
+
+  private MessageSchema schema;
+
+  public RootDeserializer(BeanDescriptor beanDescriptor, MessageSchema schema) {
+    this.beanDescriptor = beanDescriptor;
+    this.schema = schema;
+  }
+
+  public BeanDescriptor getBeanDescriptor() {
+    return beanDescriptor;
+  }
+
+  public void setBeanDescriptor(BeanDescriptor beanDescriptor) {
+    this.beanDescriptor = beanDescriptor;
+  }
+
+  public MessageSchema getSchema() {
+    return schema;
+  }
+
+  public void setSchema(MessageSchema schema) {
+    this.schema = schema;
+  }
+
+  @SuppressWarnings("unchecked")
+  public <T> T deserialize(byte[] bytes) throws IOException {
+    Input input = new ByteArrayInput(bytes, false);
+    Object instance = beanDescriptor.create();
+    schema.mergeFrom(input, instance);
+    return (T) instance;
+  }
+}
diff --git a/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/RootSerializer.java b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/RootSerializer.java
new file mode 100644
index 0000000..870c7aa
--- /dev/null
+++ b/foundations/foundation-protobuf/src/main/java/org/apache/servicecomb/foundation/protobuf/RootSerializer.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.foundation.protobuf;
+
+import java.io.IOException;
+
+import org.apache.servicecomb.foundation.protobuf.internal.schema.serializer.ProtoStreamOutput;
+
+import io.protostuff.runtime.MessageSchema;
+
+public class RootSerializer {
+  private MessageSchema schema;
+
+  public RootSerializer(MessageSchema schema) {
+    this.schema = schema;
+  }
+
+  public MessageSchema getSchema() {
+    return schema;
+  }
+
+  public void setSchema(MessageSchema schema) {
+    this.schema = schema;
+  }
+
+  /**
+   * not same to standard ProtoStuff mechanism
+   * parameter "value" of writeTo is self value, not owner
+   * @param value
+   * @return
+   * @throws Throwable
+   */
+  public byte[] serialize(Object value) throws IOException {
+    ProtoStreamOutput output = new ProtoStreamOutput();
+    schema.writeTo(output, value);
+    return output.toBytes();
+  }
+}