You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ke...@apache.org on 2024/06/24 22:38:12 UTC

(tinkerpop) 03/03: Initial implementation for transfer encoded compression

This is an automated email from the ASF dual-hosted git repository.

kenhuuu pushed a commit to branch http-compression
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git

commit 1385ee992198c4d48879d7fa7a909c1bf1ba697c
Author: Ken Hu <10...@users.noreply.github.com>
AuthorDate: Mon Jun 24 15:34:54 2024 -0700

    Initial implementation for transfer encoded compression
---
 .../tinkerpop/gremlin/driver/Channelizer.java      |  5 ++
 .../handler/HttpContentCompressionDecoder.java     | 97 ++++++++++++++++++++++
 .../gremlin/server/channel/HttpChannelizer.java    |  4 +
 .../handler/HttpContentCompressionEncoder.java     | 86 +++++++++++++++++++
 .../server/handler/HttpGremlinEndpointHandler.java |  4 +-
 5 files changed, 195 insertions(+), 1 deletion(-)

diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java
index 5d8af18a3f..f2984e3e13 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java
@@ -30,6 +30,7 @@ import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslHandler;
 import org.apache.tinkerpop.gremlin.driver.handler.GremlinResponseHandler;
+import org.apache.tinkerpop.gremlin.driver.handler.HttpContentCompressionDecoder;
 import org.apache.tinkerpop.gremlin.driver.handler.HttpGremlinRequestEncoder;
 import org.apache.tinkerpop.gremlin.driver.handler.HttpGremlinResponseStreamDecoder;
 import org.apache.tinkerpop.gremlin.driver.handler.SslCheckHandler;
@@ -167,10 +168,13 @@ public interface Channelizer extends ChannelHandler {
         private HttpGremlinRequestEncoder gremlinRequestEncoder;
         private HttpGremlinResponseStreamDecoder gremlinResponseDecoder;
 
+        private HttpContentCompressionDecoder httpCompressionDecoder;
+
         @Override
         public void init(final Connection connection) {
             super.init(connection);
 
+            httpCompressionDecoder = new HttpContentCompressionDecoder();
             gremlinRequestEncoder = new HttpGremlinRequestEncoder(cluster.getSerializer(), cluster.getRequestInterceptor(), cluster.isUserAgentOnConnectEnabled());
             gremlinResponseDecoder = new HttpGremlinResponseStreamDecoder(cluster.getSerializer(), cluster.getMaxContentLength());
         }
@@ -202,6 +206,7 @@ public interface Channelizer extends ChannelHandler {
 
             pipeline.addLast(PIPELINE_HTTP_CODEC, handler);
             pipeline.addLast(PIPELINE_HTTP_ENCODER, gremlinRequestEncoder);
+            pipeline.addLast("compression-decoder", httpCompressionDecoder);
             pipeline.addLast(PIPELINE_HTTP_DECODER, gremlinResponseDecoder);
         }
     }
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/handler/HttpContentCompressionDecoder.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/handler/HttpContentCompressionDecoder.java
new file mode 100644
index 0000000000..3351631237
--- /dev/null
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/handler/HttpContentCompressionDecoder.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.driver.handler;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.handler.codec.MessageToMessageCodec;
+import io.netty.handler.codec.MessageToMessageEncoder;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaderValues;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.util.AttributeKey;
+import io.netty.util.ReferenceCountUtil;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.Inflater;
+
+@ChannelHandler.Sharable
+public class HttpContentCompressionDecoder extends ChannelInboundHandlerAdapter {
+    private static final AttributeKey<Inflater> INFLATER = AttributeKey.valueOf("inflater");
+
+    @Override
+    public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+        Object out = msg;
+
+        if (msg instanceof HttpResponse && (hasDeflateTransferEncoding(((HttpResponse) msg).headers()))) {
+            ctx.attr(INFLATER).set(new Inflater());
+        } else if (msg instanceof HttpContent) {
+            final Inflater decompressor = ctx.attr(INFLATER).get();
+            if (decompressor != null) {
+                final ByteBuf chunk = ((HttpContent) msg).content();
+                decompressor.setInput(ByteBufUtil.getBytes(chunk));
+                final int decompressedSizeEstimate = chunk.readableBytes()*8;
+                byte[] outBuf = new byte[decompressedSizeEstimate > 10 ? decompressedSizeEstimate : 10];
+
+                int writeIdx = 0;
+                int bytesWritten = decompressor.inflate(outBuf, writeIdx, outBuf.length);
+
+                while (bytesWritten == (outBuf.length - writeIdx)) {
+                    writeIdx += bytesWritten;
+                    outBuf = Arrays.copyOf(outBuf, outBuf.length * 2);
+                    bytesWritten = decompressor.inflate(outBuf, writeIdx, outBuf.length - writeIdx);
+                }
+
+                writeIdx += bytesWritten;
+                out = ((HttpContent) msg).replace(Unpooled.wrappedBuffer(outBuf).writerIndex(writeIdx));
+                ReferenceCountUtil.release(msg);
+            }
+        }
+
+        if (out instanceof LastHttpContent) {
+            ctx.attr(INFLATER).getAndSet(null).end();
+        }
+
+        super.channelRead(ctx, out);
+    }
+
+    private boolean hasDeflateTransferEncoding(HttpHeaders headers) {
+        for (String value : headers.getAll(HttpHeaderNames.TRANSFER_ENCODING)) {
+            if ("deflate".equals(value)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java
index 93bc5aa826..2935fc95f4 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.server.auth.AllowAllAuthenticator;
 import org.apache.tinkerpop.gremlin.server.handler.AbstractAuthenticationHandler;
 import org.apache.tinkerpop.gremlin.server.handler.HttpBasicAuthenticationHandler;
 import org.apache.tinkerpop.gremlin.server.handler.HttpBasicAuthorizationHandler;
+import org.apache.tinkerpop.gremlin.server.handler.HttpContentCompressionEncoder;
 import org.apache.tinkerpop.gremlin.server.handler.HttpRequestCheckingHandler;
 import org.apache.tinkerpop.gremlin.server.handler.HttpRequestIdHandler;
 import org.apache.tinkerpop.gremlin.server.handler.HttpRequestMessageDecoder;
@@ -56,6 +57,8 @@ public class HttpChannelizer extends AbstractChannelizer {
     private HttpRequestMessageDecoder httpRequestMessageDecoder = new HttpRequestMessageDecoder(serializers);
     private HttpRequestIdHandler httpRequestIdHandler = new HttpRequestIdHandler();
 
+    private HttpContentCompressionEncoder httpContentEncoder = new HttpContentCompressionEncoder();
+
     @Override
     public void init(final ServerGremlinExecutor serverGremlinExecutor) {
         super.init(serverGremlinExecutor);
@@ -81,6 +84,7 @@ public class HttpChannelizer extends AbstractChannelizer {
         pipeline.addLast(PIPELINE_HTTP_AGGREGATOR, aggregator);
         pipeline.addLast("http-request-checker", httpRequestCheckingHandler);
         pipeline.addLast("http-user-agent-handler", new HttpUserAgentHandler());
+        pipeline.addLast("http-compression-encoder", httpContentEncoder);
 
         if (authenticator != null) {
             // Cannot add the same handler instance multiple times unless
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpContentCompressionEncoder.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpContentCompressionEncoder.java
new file mode 100644
index 0000000000..c7f96fffa8
--- /dev/null
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpContentCompressionEncoder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.server.handler;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.handler.codec.MessageToMessageCodec;
+import io.netty.handler.codec.MessageToMessageEncoder;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.util.AttributeKey;
+import io.netty.util.ReferenceCountUtil;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.Deflater;
+
+@ChannelHandler.Sharable
+public class HttpContentCompressionEncoder extends ChannelOutboundHandlerAdapter {
+    private static final AttributeKey<Deflater> DEFLATER = AttributeKey.valueOf("deflater");
+
+    @Override
+    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
+        Object out = msg;
+        int writeIdx = 0;
+        int bytesWritten = 0;
+
+        if (msg instanceof HttpResponse && (((HttpResponse) msg).headers().contains(HttpHeaderNames.TRANSFER_ENCODING))) {
+            ctx.attr(DEFLATER).set(new Deflater());
+        } else if (msg instanceof HttpContent) {
+            final ByteBuf chunk = ((HttpContent) msg).content();
+            final Deflater compressor = ctx.attr(DEFLATER).get();
+
+            if (compressor != null) {
+                compressor.setInput(ByteBufUtil.getBytes(chunk));
+
+                if (msg instanceof LastHttpContent) compressor.finish();
+
+                final int compressedSizeEstimate = chunk.readableBytes() / 8;
+                byte[] outBuf = new byte[compressedSizeEstimate > 10 ? compressedSizeEstimate : 10];
+                bytesWritten = compressor.deflate(outBuf, writeIdx, outBuf.length, Deflater.SYNC_FLUSH);
+                while (bytesWritten == (outBuf.length - writeIdx)) {
+                    writeIdx += bytesWritten;
+                    outBuf = Arrays.copyOf(outBuf, outBuf.length * 2);
+                    bytesWritten = compressor.deflate(outBuf, writeIdx, outBuf.length - writeIdx, Deflater.SYNC_FLUSH);
+                }
+                writeIdx += bytesWritten;
+                out = ((HttpContent) msg).replace(Unpooled.wrappedBuffer(outBuf).writerIndex(writeIdx));
+                ReferenceCountUtil.release(msg);
+            }
+        }
+
+        if (out instanceof LastHttpContent) {
+            ctx.attr(DEFLATER).getAndSet(null).end();
+        }
+
+        super.write(ctx, out, promise);
+    }
+}
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
index 711a0d87ae..cf4fa913a4 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
@@ -93,6 +93,7 @@ import java.util.stream.Stream;
 import static com.codahale.metrics.MetricRegistry.name;
 import static io.netty.handler.codec.http.HttpHeaderNames.TRANSFER_ENCODING;
 import static io.netty.handler.codec.http.HttpHeaderValues.CHUNKED;
+import static io.netty.handler.codec.http.HttpHeaderValues.DEFLATE;
 import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
 import static io.netty.handler.codec.http.HttpResponseStatus.OK;
 import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
@@ -198,7 +199,8 @@ public class HttpGremlinEndpointHandler extends SimpleChannelInboundHandler<Requ
                 // Send back the 200 OK response header here since the response is always chunk transfer encoded. Any
                 // failures that follow this will show up in the response body instead.
                 final HttpResponse responseHeader = new DefaultHttpResponse(HTTP_1_1, OK);
-                responseHeader.headers().set(TRANSFER_ENCODING, CHUNKED);
+                responseHeader.headers().add(TRANSFER_ENCODING, DEFLATE);
+                responseHeader.headers().add(TRANSFER_ENCODING, CHUNKED);
                 responseHeader.headers().set(HttpHeaderNames.CONTENT_TYPE, serializer.getValue0());
                 ctx.writeAndFlush(responseHeader);
                 ctx.channel().attr(StateKey.HTTP_RESPONSE_SENT).set(true);