You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@dubbo.apache.org by GitBox <gi...@apache.org> on 2018/07/09 09:53:04 UTC
[GitHub] playaround88 closed pull request #1994: avro serialization implement
playaround88 closed pull request #1994: avro serialization implement
URL: https://github.com/apache/incubator-dubbo/pull/1994
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bom/pom.xml b/bom/pom.xml
index 03e2eb497f..d24530429c 100644
--- a/bom/pom.xml
+++ b/bom/pom.xml
@@ -278,6 +278,11 @@
<artifactId>dubbo-serialization-kryo</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.dubbo</groupId>
+ <artifactId>dubbo-serialization-avro</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo-bootstrap</artifactId>
diff --git a/dependencies-bom/pom.xml b/dependencies-bom/pom.xml
index ddaefce8c1..e61a06f9ab 100644
--- a/dependencies-bom/pom.xml
+++ b/dependencies-bom/pom.xml
@@ -97,6 +97,7 @@
<kryo_version>4.0.1</kryo_version>
<kryo_serializers_version>0.42</kryo_serializers_version>
<fst_version>2.48-jdk-6</fst_version>
+ <avro_version>1.8.2</avro_version>
<rs_api_version>2.0</rs_api_version>
<resteasy_version>3.0.19.Final</resteasy_version>
@@ -159,6 +160,11 @@
<artifactId>fastjson</artifactId>
<version>${fastjson_version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro_version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
diff --git a/dubbo-serialization/dubbo-serialization-avro/pom.xml b/dubbo-serialization/dubbo-serialization-avro/pom.xml
new file mode 100644
index 0000000000..cede5252d7
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/pom.xml
@@ -0,0 +1,24 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.dubbo</groupId>
+ <artifactId>dubbo-serialization</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>dubbo-serialization-avro</artifactId>
+ <name>${project.artifactId}</name>
+ <description>The common module of dubbo project</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.dubbo</groupId>
+ <artifactId>dubbo-serialization-api</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroObjectInput.java b/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroObjectInput.java
new file mode 100644
index 0000000000..f498f55936
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroObjectInput.java
@@ -0,0 +1,100 @@
+package org.apache.dubbo.common.serialize.avro;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.avro.io.BinaryDecoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.avro.util.Utf8;
+import org.apache.dubbo.common.serialize.ObjectInput;
+
+public class AvroObjectInput implements ObjectInput{
+ private DecoderFactory decoderFactory=DecoderFactory.get();;
+ private BinaryDecoder decoder;
+
+ public AvroObjectInput(InputStream in){
+ decoder=decoderFactory.binaryDecoder(in, null);
+ }
+
+ @Override
+ public boolean readBool() throws IOException {
+ return decoder.readBoolean();
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ byte[] bytes=new byte[1];
+ decoder.readFixed(bytes);
+ return bytes[0];
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ return (short) decoder.readInt();
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ return decoder.readInt();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ return decoder.readLong();
+ }
+
+ @Override
+ public float readFloat() throws IOException {
+ return decoder.readFloat();
+ }
+
+ @Override
+ public double readDouble() throws IOException {
+ return decoder.readDouble();
+ }
+
+ @Override
+ public String readUTF() throws IOException {
+ Utf8 result= new Utf8();
+ result=decoder.readString(result);
+ return result.toString();
+ }
+
+ @Override
+ public byte[] readBytes() throws IOException {
+ String resultStr = decoder.readString();
+ return resultStr.getBytes("utf8");
+ }
+
+ /**
+ * will lost all attribute
+ */
+ @Override
+ public Object readObject() throws IOException, ClassNotFoundException {
+ ReflectDatumReader<Object> reader = new ReflectDatumReader<>(Object.class);
+ return reader.read(null, decoder);
+ }
+
+ @Override
+ @SuppressWarnings(value={"unchecked"})
+ public <T> T readObject(Class<T> cls) throws IOException, ClassNotFoundException {
+ //Map interface class change to HashMap implement
+ if(cls==Map.class){
+ cls=(Class<T>) HashMap.class;
+ }
+
+ ReflectDatumReader<T> reader = new ReflectDatumReader<>(cls);
+ return reader.read(null, decoder);
+ }
+
+ @Override
+ public <T> T readObject(Class<T> cls, Type type) throws IOException, ClassNotFoundException {
+ ReflectDatumReader<T> reader = new ReflectDatumReader<>(cls);
+ return reader.read(null, decoder);
+ }
+
+}
diff --git a/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroObjectOutput.java b/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroObjectOutput.java
new file mode 100644
index 0000000000..3c8f102788
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroObjectOutput.java
@@ -0,0 +1,88 @@
+package org.apache.dubbo.common.serialize.avro;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.reflect.ReflectDatumWriter;
+import org.apache.avro.util.Utf8;
+import org.apache.dubbo.common.serialize.ObjectOutput;
+
+public class AvroObjectOutput implements ObjectOutput{
+ private EncoderFactory encoderFactory=EncoderFactory.get();
+ private BinaryEncoder encoder;
+
+ public AvroObjectOutput(OutputStream out) {
+ encoder=encoderFactory.binaryEncoder(out, null);
+ }
+
+ @Override
+ public void writeBool(boolean v) throws IOException {
+ encoder.writeBoolean(v);
+ }
+
+ @Override
+ public void writeByte(byte v) throws IOException {
+ encoder.writeFixed(new byte[]{v});
+ }
+
+ @Override
+ public void writeShort(short v) throws IOException {
+ encoder.writeInt(v);
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ encoder.writeInt(v);
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ encoder.writeLong(v);
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ encoder.writeFloat(v);
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ encoder.writeDouble(v);
+ }
+
+ @Override
+ public void writeUTF(String v) throws IOException {
+ encoder.writeString(new Utf8(v));
+ }
+
+ @Override
+ public void writeBytes(byte[] v) throws IOException {
+ encoder.writeString(new String(v, "utf8"));
+ }
+
+ @Override
+ public void writeBytes(byte[] v, int off, int len) throws IOException {
+ byte[] v2 = Arrays.copyOfRange(v, off, off+len);
+ encoder.writeString(new String(v2, "utf8"));
+ }
+
+ @Override
+ public void flushBuffer() throws IOException {
+ encoder.flush();
+ }
+
+ @Override
+ @SuppressWarnings(value = { "rawtypes", "unchecked" })
+ public void writeObject(Object obj) throws IOException {
+ if(obj==null) {
+ encoder.writeNull();
+ return ;
+ }
+ ReflectDatumWriter dd=new ReflectDatumWriter<>(obj.getClass());
+ dd.write(obj, encoder);
+ }
+
+}
diff --git a/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroSerialization.java b/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroSerialization.java
new file mode 100644
index 0000000000..9b86f1ce7a
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/src/main/java/org/apache/dubbo/common/serialize/avro/AvroSerialization.java
@@ -0,0 +1,34 @@
+package org.apache.dubbo.common.serialize.avro;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.dubbo.common.URL;
+import org.apache.dubbo.common.serialize.ObjectInput;
+import org.apache.dubbo.common.serialize.ObjectOutput;
+import org.apache.dubbo.common.serialize.Serialization;
+
+public class AvroSerialization implements Serialization{
+
+ @Override
+ public byte getContentTypeId() {
+ return 10;
+ }
+
+ @Override
+ public String getContentType() {
+ return "avro/binary";
+ }
+
+ @Override
+ public ObjectOutput serialize(URL url, OutputStream output) throws IOException {
+ return new AvroObjectOutput(output);
+ }
+
+ @Override
+ public ObjectInput deserialize(URL url, InputStream input) throws IOException {
+ return new AvroObjectInput(input);
+ }
+
+}
diff --git a/dubbo-serialization/dubbo-serialization-avro/src/main/resources/META-INF/dubbo/com.alibaba.dubbo.common.serialize.Serialization b/dubbo-serialization/dubbo-serialization-avro/src/main/resources/META-INF/dubbo/com.alibaba.dubbo.common.serialize.Serialization
new file mode 100644
index 0000000000..d8cba3e1ba
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/src/main/resources/META-INF/dubbo/com.alibaba.dubbo.common.serialize.Serialization
@@ -0,0 +1 @@
+avro=org.apache.dubbo.common.serialize.avro.AvroSerialization
\ No newline at end of file
diff --git a/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/AvroObjectInputOutputTest.java b/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/AvroObjectInputOutputTest.java
new file mode 100644
index 0000000000..7e884d7147
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/AvroObjectInputOutputTest.java
@@ -0,0 +1,180 @@
+package org.apache.dubbo.common.serialize.avro;
+
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.hamcrest.core.IsNull.nullValue;
+import static org.junit.Assert.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+
+import org.apache.dubbo.common.serialize.avro.model.Person;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AvroObjectInputOutputTest {
+ private AvroObjectInput avroObjectInput;
+ private AvroObjectOutput avroObjectOutput;
+
+ private PipedOutputStream pos;
+ private PipedInputStream pis;
+
+ @Before
+ public void setup() throws IOException {
+ pis = new PipedInputStream();
+ pos = new PipedOutputStream();
+ pis.connect(pos);
+
+ avroObjectOutput = new AvroObjectOutput(pos);
+ avroObjectInput = new AvroObjectInput(pis);
+ }
+
+ @After
+ public void clean() throws IOException {
+ if (pos != null) {
+ pos.close();
+ pos = null;
+ }
+ if (pis != null) {
+ pis.close();
+ pis = null;
+ }
+ }
+
+ @Test
+ public void testWriteReadBool() throws IOException, InterruptedException {
+ avroObjectOutput.writeBool(true);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ boolean result = avroObjectInput.readBool();
+ assertThat(result, is(true));
+ }
+
+ @Test
+ public void testWriteReadByte() throws IOException {
+ avroObjectOutput.writeByte((byte) 'a');
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ Byte result = avroObjectInput.readByte();
+
+ assertThat(result, is((byte) 'a'));
+ }
+
+ @Test
+ public void testWriteReadBytes() throws IOException {
+ avroObjectOutput.writeBytes("123456".getBytes());
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ byte[] result = avroObjectInput.readBytes();
+
+ assertThat(result, is("123456".getBytes()));
+ }
+
+ @Test
+ public void testWriteReadShort() throws IOException {
+ avroObjectOutput.writeShort((short) 1);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ short result = avroObjectInput.readShort();
+
+ assertThat(result, is((short) 1));
+ }
+
+ @Test
+ public void testWriteReadInt() throws IOException {
+ avroObjectOutput.writeInt(1);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ Integer result = avroObjectInput.readInt();
+
+ assertThat(result, is(1));
+ }
+
+ @Test
+ public void testReadDouble() throws IOException {
+ avroObjectOutput.writeDouble(3.14d);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ Double result = avroObjectInput.readDouble();
+
+ assertThat(result, is(3.14d));
+ }
+
+ @Test
+ public void testReadLong() throws IOException {
+ avroObjectOutput.writeLong(10L);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ Long result = avroObjectInput.readLong();
+
+ assertThat(result, is(10L));
+ }
+
+ @Test
+ public void testWriteReadFloat() throws IOException {
+ avroObjectOutput.writeFloat(1.66f);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ Float result = avroObjectInput.readFloat();
+
+ assertThat(result, is(1.66F));
+ }
+
+ @Test
+ public void testWriteReadUTF() throws IOException {
+ avroObjectOutput.writeUTF("wording");
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ String result = avroObjectInput.readUTF();
+
+ assertThat(result, is("wording"));
+ }
+
+ @Test
+ public void testWriteReadObject() throws IOException, ClassNotFoundException {
+ Person p = new Person();
+ p.setAge(30);
+ p.setName("abc");
+
+ avroObjectOutput.writeObject(p);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ Person result = avroObjectInput.readObject(Person.class);
+
+ assertThat(result, not(nullValue()));
+ assertThat(result.getName(), is("abc"));
+ assertThat(result.getAge(), is(30));
+ }
+
+ @Test
+ public void testWriteReadObjectWithoutClass() throws IOException, ClassNotFoundException {
+ Person p = new Person();
+ p.setAge(30);
+ p.setName("abc");
+
+ avroObjectOutput.writeObject(p);
+ avroObjectOutput.flushBuffer();
+ pos.close();
+
+ //这里会丢失所有信息
+ Object result = avroObjectInput.readObject();
+
+ assertThat(result, not(nullValue()));
+// assertThat(result.getName(), is("abc"));
+// assertThat(result.getAge(), is(30));
+ }
+}
diff --git a/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/AvroSerializationTest.java b/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/AvroSerializationTest.java
new file mode 100644
index 0000000000..be1828df3d
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/AvroSerializationTest.java
@@ -0,0 +1,46 @@
+package org.apache.dubbo.common.serialize.avro;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.dubbo.common.serialize.ObjectInput;
+import org.apache.dubbo.common.serialize.ObjectOutput;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AvroSerializationTest {
+ private AvroSerialization avroSerialization;
+
+ @Before
+ public void setUp() {
+ this.avroSerialization = new AvroSerialization();
+ }
+
+ @Test
+ public void testContentType() {
+ assertThat(avroSerialization.getContentType(), is("avro/binary"));
+ }
+
+ @Test
+ public void testContentTypeId() {
+ assertThat(avroSerialization.getContentTypeId(), is((byte) 10));
+ }
+
+ @Test
+ public void testObjectOutput() throws IOException {
+ ObjectOutput objectOutput = avroSerialization.serialize(null, mock(OutputStream.class));
+ assertThat(objectOutput, Matchers.<ObjectOutput>instanceOf(AvroObjectOutput.class));
+ }
+
+ @Test
+ public void testObjectInput() throws IOException {
+ ObjectInput objectInput = avroSerialization.deserialize(null, mock(InputStream.class));
+ assertThat(objectInput, Matchers.<ObjectInput>instanceOf(AvroObjectInput.class));
+ }
+}
diff --git a/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/model/Person.java b/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/model/Person.java
new file mode 100644
index 0000000000..8701a4fdbe
--- /dev/null
+++ b/dubbo-serialization/dubbo-serialization-avro/src/test/java/org/apache/dubbo/common/serialize/avro/model/Person.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.serialize.avro.model;
+
+public class Person {
+ private String name;
+ private int age;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public int getAge() {
+ return age;
+ }
+
+ public void setAge(int age) {
+ this.age = age;
+ }
+}
diff --git a/dubbo-serialization/pom.xml b/dubbo-serialization/pom.xml
index 3201877ea5..a4cea4e8b5 100644
--- a/dubbo-serialization/pom.xml
+++ b/dubbo-serialization/pom.xml
@@ -35,5 +35,6 @@
<module>dubbo-serialization-kryo</module>
<module>dubbo-serialization-fst</module>
<module>dubbo-serialization-jdk</module>
+ <module>dubbo-serialization-avro</module>
</modules>
</project>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@dubbo.apache.org
For additional commands, e-mail: notifications-help@dubbo.apache.org