You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2021/08/11 13:39:47 UTC

[flink-statefun] branch master updated: [FLINK-23712] Improve Protobuf serialization performance

This is an automated email from the ASF dual-hosted git repository.

tzulitai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-statefun.git


The following commit(s) were added to refs/heads/master by this push:
     new 2d5d6b2  [FLINK-23712] Improve Protobuf serialization performance
2d5d6b2 is described below

commit 2d5d6b249ad7df93c3612b6e599b86b697151e30
Author: Igal Shilman <ig...@gmail.com>
AuthorDate: Tue Aug 10 18:13:19 2021 +0200

    [FLINK-23712] Improve Protobuf serialization performance
    
    Profiling allocations shows that around 25% of the allocations happens
    while parsing Protobuf, this happens because
    the CodedInputStream/CodedOutputStream allocate a 4kb buffer on construction.
    This can be easily avoided by caching and reusing them.
    
    This closes #251.
---
 .../statefun/flink/common/protobuf/ProtobufSerializer.java   | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/statefun-flink/statefun-flink-common/src/main/java/org/apache/flink/statefun/flink/common/protobuf/ProtobufSerializer.java b/statefun-flink/statefun-flink-common/src/main/java/org/apache/flink/statefun/flink/common/protobuf/ProtobufSerializer.java
index d314f59..e99888f 100644
--- a/statefun-flink/statefun-flink-common/src/main/java/org/apache/flink/statefun/flink/common/protobuf/ProtobufSerializer.java
+++ b/statefun-flink/statefun-flink-common/src/main/java/org/apache/flink/statefun/flink/common/protobuf/ProtobufSerializer.java
@@ -17,6 +17,8 @@
  */
 package org.apache.flink.statefun.flink.common.protobuf;
 
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.DescriptorProtos.FileDescriptorSet;
 import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Message;
@@ -32,7 +34,9 @@ import org.apache.flink.statefun.flink.common.generated.ProtobufSerializerSnapsh
 public final class ProtobufSerializer<M extends Message> {
 
   private final OutputStreamView output;
+  private final CodedOutputStream codedOutputStream;
   private final InputStreamView input;
+  private final CodedInputStream codedInputStream;
   private final Parser<M> parser;
   private final ProtobufSerializerSnapshot snapshot;
 
@@ -48,6 +52,8 @@ public final class ProtobufSerializer<M extends Message> {
     this.snapshot = Objects.requireNonNull(snapshot);
     this.input = new InputStreamView();
     this.output = new OutputStreamView();
+    this.codedInputStream = CodedInputStream.newInstance(input);
+    this.codedOutputStream = CodedOutputStream.newInstance(output);
   }
 
   public void serialize(M record, DataOutputView target) throws IOException {
@@ -56,7 +62,8 @@ public final class ProtobufSerializer<M extends Message> {
 
     output.set(target);
     try {
-      record.writeTo(output);
+      record.writeTo(codedOutputStream);
+      codedOutputStream.flush();
     } finally {
       output.done();
     }
@@ -65,8 +72,9 @@ public final class ProtobufSerializer<M extends Message> {
   public M deserialize(DataInputView source) throws IOException {
     final int serializedSize = source.readInt();
     input.set(source, serializedSize);
+    codedInputStream.resetSizeCounter();
     try {
-      return parser.parseFrom(input);
+      return parser.parseFrom(codedInputStream);
     } finally {
       input.done();
     }