You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ke...@apache.org on 2024/06/24 22:38:12 UTC
(tinkerpop) 03/03: Initial implementation for transfer encoded compression
This is an automated email from the ASF dual-hosted git repository.
kenhuuu pushed a commit to branch http-compression
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 1385ee992198c4d48879d7fa7a909c1bf1ba697c
Author: Ken Hu <10...@users.noreply.github.com>
AuthorDate: Mon Jun 24 15:34:54 2024 -0700
Initial implementation for transfer encoded compression
---
.../tinkerpop/gremlin/driver/Channelizer.java | 5 ++
.../handler/HttpContentCompressionDecoder.java | 97 ++++++++++++++++++++++
.../gremlin/server/channel/HttpChannelizer.java | 4 +
.../handler/HttpContentCompressionEncoder.java | 86 +++++++++++++++++++
.../server/handler/HttpGremlinEndpointHandler.java | 4 +-
5 files changed, 195 insertions(+), 1 deletion(-)
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java
index 5d8af18a3f..f2984e3e13 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Channelizer.java
@@ -30,6 +30,7 @@ import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslHandler;
import org.apache.tinkerpop.gremlin.driver.handler.GremlinResponseHandler;
+import org.apache.tinkerpop.gremlin.driver.handler.HttpContentCompressionDecoder;
import org.apache.tinkerpop.gremlin.driver.handler.HttpGremlinRequestEncoder;
import org.apache.tinkerpop.gremlin.driver.handler.HttpGremlinResponseStreamDecoder;
import org.apache.tinkerpop.gremlin.driver.handler.SslCheckHandler;
@@ -167,10 +168,13 @@ public interface Channelizer extends ChannelHandler {
private HttpGremlinRequestEncoder gremlinRequestEncoder;
private HttpGremlinResponseStreamDecoder gremlinResponseDecoder;
+ private HttpContentCompressionDecoder httpCompressionDecoder;
+
@Override
public void init(final Connection connection) {
super.init(connection);
+ httpCompressionDecoder = new HttpContentCompressionDecoder();
gremlinRequestEncoder = new HttpGremlinRequestEncoder(cluster.getSerializer(), cluster.getRequestInterceptor(), cluster.isUserAgentOnConnectEnabled());
gremlinResponseDecoder = new HttpGremlinResponseStreamDecoder(cluster.getSerializer(), cluster.getMaxContentLength());
}
@@ -202,6 +206,7 @@ public interface Channelizer extends ChannelHandler {
pipeline.addLast(PIPELINE_HTTP_CODEC, handler);
pipeline.addLast(PIPELINE_HTTP_ENCODER, gremlinRequestEncoder);
+ pipeline.addLast("compression-decoder", httpCompressionDecoder);
pipeline.addLast(PIPELINE_HTTP_DECODER, gremlinResponseDecoder);
}
}
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/handler/HttpContentCompressionDecoder.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/handler/HttpContentCompressionDecoder.java
new file mode 100644
index 0000000000..3351631237
--- /dev/null
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/handler/HttpContentCompressionDecoder.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.driver.handler;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.handler.codec.MessageToMessageCodec;
+import io.netty.handler.codec.MessageToMessageEncoder;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaderValues;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.util.AttributeKey;
+import io.netty.util.ReferenceCountUtil;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.Inflater;
+
+@ChannelHandler.Sharable
+public class HttpContentCompressionDecoder extends ChannelInboundHandlerAdapter {
+ private static final AttributeKey<Inflater> INFLATER = AttributeKey.valueOf("inflater");
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
+ Object out = msg;
+
+ if (msg instanceof HttpResponse && (hasDeflateTransferEncoding(((HttpResponse) msg).headers()))) {
+ ctx.attr(INFLATER).set(new Inflater());
+ } else if (msg instanceof HttpContent) {
+ final Inflater decompressor = ctx.attr(INFLATER).get();
+ if (decompressor != null) {
+ final ByteBuf chunk = ((HttpContent) msg).content();
+ decompressor.setInput(ByteBufUtil.getBytes(chunk));
+ final int decompressedSizeEstimate = chunk.readableBytes()*8;
+ byte[] outBuf = new byte[decompressedSizeEstimate > 10 ? decompressedSizeEstimate : 10];
+
+ int writeIdx = 0;
+ int bytesWritten = decompressor.inflate(outBuf, writeIdx, outBuf.length);
+
+ while (bytesWritten == (outBuf.length - writeIdx)) {
+ writeIdx += bytesWritten;
+ outBuf = Arrays.copyOf(outBuf, outBuf.length * 2);
+ bytesWritten = decompressor.inflate(outBuf, writeIdx, outBuf.length - writeIdx);
+ }
+
+ writeIdx += bytesWritten;
+ out = ((HttpContent) msg).replace(Unpooled.wrappedBuffer(outBuf).writerIndex(writeIdx));
+ ReferenceCountUtil.release(msg);
+ }
+ }
+
+ if (out instanceof LastHttpContent) {
+ ctx.attr(INFLATER).getAndSet(null).end();
+ }
+
+ super.channelRead(ctx, out);
+ }
+
+ private boolean hasDeflateTransferEncoding(HttpHeaders headers) {
+ for (String value : headers.getAll(HttpHeaderNames.TRANSFER_ENCODING)) {
+ if ("deflate".equals(value)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+}
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java
index 93bc5aa826..2935fc95f4 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/channel/HttpChannelizer.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.server.auth.AllowAllAuthenticator;
import org.apache.tinkerpop.gremlin.server.handler.AbstractAuthenticationHandler;
import org.apache.tinkerpop.gremlin.server.handler.HttpBasicAuthenticationHandler;
import org.apache.tinkerpop.gremlin.server.handler.HttpBasicAuthorizationHandler;
+import org.apache.tinkerpop.gremlin.server.handler.HttpContentCompressionEncoder;
import org.apache.tinkerpop.gremlin.server.handler.HttpRequestCheckingHandler;
import org.apache.tinkerpop.gremlin.server.handler.HttpRequestIdHandler;
import org.apache.tinkerpop.gremlin.server.handler.HttpRequestMessageDecoder;
@@ -56,6 +57,8 @@ public class HttpChannelizer extends AbstractChannelizer {
private HttpRequestMessageDecoder httpRequestMessageDecoder = new HttpRequestMessageDecoder(serializers);
private HttpRequestIdHandler httpRequestIdHandler = new HttpRequestIdHandler();
+ private HttpContentCompressionEncoder httpContentEncoder = new HttpContentCompressionEncoder();
+
@Override
public void init(final ServerGremlinExecutor serverGremlinExecutor) {
super.init(serverGremlinExecutor);
@@ -81,6 +84,7 @@ public class HttpChannelizer extends AbstractChannelizer {
pipeline.addLast(PIPELINE_HTTP_AGGREGATOR, aggregator);
pipeline.addLast("http-request-checker", httpRequestCheckingHandler);
pipeline.addLast("http-user-agent-handler", new HttpUserAgentHandler());
+ pipeline.addLast("http-compression-encoder", httpContentEncoder);
if (authenticator != null) {
// Cannot add the same handler instance multiple times unless
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpContentCompressionEncoder.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpContentCompressionEncoder.java
new file mode 100644
index 0000000000..c7f96fffa8
--- /dev/null
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpContentCompressionEncoder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.server.handler;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.handler.codec.MessageToMessageCodec;
+import io.netty.handler.codec.MessageToMessageEncoder;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpObject;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.util.AttributeKey;
+import io.netty.util.ReferenceCountUtil;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.Deflater;
+
+@ChannelHandler.Sharable
+public class HttpContentCompressionEncoder extends ChannelOutboundHandlerAdapter {
+ private static final AttributeKey<Deflater> DEFLATER = AttributeKey.valueOf("deflater");
+
+ @Override
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
+ Object out = msg;
+ int writeIdx = 0;
+ int bytesWritten = 0;
+
+ if (msg instanceof HttpResponse && (((HttpResponse) msg).headers().contains(HttpHeaderNames.TRANSFER_ENCODING))) {
+ ctx.attr(DEFLATER).set(new Deflater());
+ } else if (msg instanceof HttpContent) {
+ final ByteBuf chunk = ((HttpContent) msg).content();
+ final Deflater compressor = ctx.attr(DEFLATER).get();
+
+ if (compressor != null) {
+ compressor.setInput(ByteBufUtil.getBytes(chunk));
+
+ if (msg instanceof LastHttpContent) compressor.finish();
+
+ final int compressedSizeEstimate = chunk.readableBytes() / 8;
+ byte[] outBuf = new byte[compressedSizeEstimate > 10 ? compressedSizeEstimate : 10];
+ bytesWritten = compressor.deflate(outBuf, writeIdx, outBuf.length, Deflater.SYNC_FLUSH);
+ while (bytesWritten == (outBuf.length - writeIdx)) {
+ writeIdx += bytesWritten;
+ outBuf = Arrays.copyOf(outBuf, outBuf.length * 2);
+ bytesWritten = compressor.deflate(outBuf, writeIdx, outBuf.length - writeIdx, Deflater.SYNC_FLUSH);
+ }
+ writeIdx += bytesWritten;
+ out = ((HttpContent) msg).replace(Unpooled.wrappedBuffer(outBuf).writerIndex(writeIdx));
+ ReferenceCountUtil.release(msg);
+ }
+ }
+
+ if (out instanceof LastHttpContent) {
+ ctx.attr(DEFLATER).getAndSet(null).end();
+ }
+
+ super.write(ctx, out, promise);
+ }
+}
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
index 711a0d87ae..cf4fa913a4 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/handler/HttpGremlinEndpointHandler.java
@@ -93,6 +93,7 @@ import java.util.stream.Stream;
import static com.codahale.metrics.MetricRegistry.name;
import static io.netty.handler.codec.http.HttpHeaderNames.TRANSFER_ENCODING;
import static io.netty.handler.codec.http.HttpHeaderValues.CHUNKED;
+import static io.netty.handler.codec.http.HttpHeaderValues.DEFLATE;
import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
import static io.netty.handler.codec.http.HttpResponseStatus.OK;
import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
@@ -198,7 +199,8 @@ public class HttpGremlinEndpointHandler extends SimpleChannelInboundHandler<Requ
// Send back the 200 OK response header here since the response is always chunk transfer encoded. Any
// failures that follow this will show up in the response body instead.
final HttpResponse responseHeader = new DefaultHttpResponse(HTTP_1_1, OK);
- responseHeader.headers().set(TRANSFER_ENCODING, CHUNKED);
+ responseHeader.headers().add(TRANSFER_ENCODING, DEFLATE);
+ responseHeader.headers().add(TRANSFER_ENCODING, CHUNKED);
responseHeader.headers().set(HttpHeaderNames.CONTENT_TYPE, serializer.getValue0());
ctx.writeAndFlush(responseHeader);
ctx.channel().attr(StateKey.HTTP_RESPONSE_SENT).set(true);