You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cxf.apache.org by re...@apache.org on 2024/11/30 14:19:36 UTC

(cxf) 10/11: CXF-8931: HttpClientHTTPConduit can't disable the http chunk mode (#2018)

This is an automated email from the ASF dual-hosted git repository.

reta pushed a commit to branch 3.6.x-fixes
in repository https://gitbox.apache.org/repos/asf/cxf.git

commit 81e4dfdffa777408dc7856dd91dd1115b48764d5
Author: Andriy Redko <dr...@gmail.com>
AuthorDate: Sat Nov 30 09:18:38 2024 -0500

    CXF-8931: HttpClientHTTPConduit can't disable the http chunk mode (#2018)
---
 .../cxf/transport/http/HttpClientHTTPConduit.java  | 214 ++++++++++++++++++---
 .../org/apache/cxf/systest/jaxrs/FileStore.java    | 128 ++++++++++++
 .../apache/cxf/systest/jaxrs/FileStoreServer.java  |  47 +++++
 .../cxf/systest/jaxrs/JAXRSClientChunkingTest.java | 117 +++++++++++
 4 files changed, 479 insertions(+), 27 deletions(-)

diff --git a/rt/transports/http/src/main/java/org/apache/cxf/transport/http/HttpClientHTTPConduit.java b/rt/transports/http/src/main/java/org/apache/cxf/transport/http/HttpClientHTTPConduit.java
index a59bc3e6a1..f3190348d9 100644
--- a/rt/transports/http/src/main/java/org/apache/cxf/transport/http/HttpClientHTTPConduit.java
+++ b/rt/transports/http/src/main/java/org/apache/cxf/transport/http/HttpClientHTTPConduit.java
@@ -18,6 +18,9 @@
  */
 package org.apache.cxf.transport.http;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -517,11 +520,11 @@ public class HttpClientHTTPConduit extends URLConnectionHTTPConduit {
     static class HttpClientPipedOutputStream extends PipedOutputStream {
         HttpClientWrappedOutputStream stream;
         HTTPClientPolicy csPolicy;
-        HttpClientBodyPublisher publisher;
-        HttpClientPipedOutputStream(HttpClientWrappedOutputStream s,
+        CloseableBodyPublisher publisher;
+        HttpClientPipedOutputStream(HttpClientWrappedOutputStream s, 
                                     PipedInputStream pin,
                                     HTTPClientPolicy cp,
-                                    HttpClientBodyPublisher bp) throws IOException {
+                                    CloseableBodyPublisher bp) throws IOException {
             super(pin);
             stream = s;
             csPolicy = cp;
@@ -592,26 +595,27 @@ public class HttpClientHTTPConduit extends URLConnectionHTTPConduit {
             }
         }
     }
-    private static final class InputStreamSupplier implements Supplier<InputStream> {
-        final InputStream in;
-        InputStreamSupplier(InputStream i) {
-            in = i;
-        }
 
-        public InputStream get() {
-            return in;
-        }
+    /**
+     * The interface for {@link BodyPublisher}s that implement {@link Closeable} as well.
+     */
+    private interface CloseableBodyPublisher extends BodyPublisher, Closeable {
     }
-    private static final class HttpClientBodyPublisher implements BodyPublisher {
-        PipedInputStream pin;
-        HttpClientWrappedOutputStream  stream;
-        long contentLen;
+    
+    /**
+     * The {@link BodyPublisher} that wraps around the output stream.
+     */
+    private static final class HttpClientBodyPublisher implements CloseableBodyPublisher {
+        private Supplier<InputStream> pin;
+        private HttpClientWrappedOutputStream stream;
+        private long contentLen;
 
-        private HttpClientBodyPublisher(HttpClientWrappedOutputStream s, PipedInputStream pin) {
+        private HttpClientBodyPublisher(HttpClientWrappedOutputStream s, Supplier<InputStream> pin) {
             this.stream = s;
             this.pin = pin;
         }
-        synchronized void close() {
+
+        public synchronized void close() {
             if (stream != null) {
                 contentLen = stream.contentLen;
                 stream = null;
@@ -630,7 +634,7 @@ public class HttpClientHTTPConduit extends URLConnectionHTTPConduit {
                     if (stream != null) {
                         contentLen = stream.contentLen;
                     }
-                    BodyPublishers.ofInputStream(new InputStreamSupplier(pin)).subscribe(subscriber);
+                    BodyPublishers.ofInputStream(pin).subscribe(subscriber);
                     stream = null;
                     pin = null;
                     return;
@@ -647,7 +651,155 @@ public class HttpClientHTTPConduit extends URLConnectionHTTPConduit {
             return contentLen;
         }
     }
-    class HttpClientWrappedOutputStream extends WrappedOutputStream {
+
+    /**
+     * The {@link BodyPublisher} that awaits for the output stream to be fully flushed (closed) 
+     * so the content length becomes known (sized). It is used when the chunked transfer is not allowed
+     * but the content length is not specified up-front.
+     */
+    private static final class HttpClientSizedBodyPublisher implements CloseableBodyPublisher {
+        private HTTPClientPolicy csPolicy;
+        private Supplier<ByteArrayInputStream> pin;
+        private HttpClientWrappedOutputStream stream;
+        private long contentLen;
+
+        private HttpClientSizedBodyPublisher(HttpClientWrappedOutputStream s, HTTPClientPolicy cs,
+                Supplier<ByteArrayInputStream> pin) {
+            this.stream = s;
+            this.csPolicy = cs;
+            this.pin = pin;
+        }
+
+        public synchronized void close() {
+            if (stream != null) {
+                contentLen = stream.contentLen;
+                stream = null;
+            }
+        }
+
+        @Override
+        public synchronized void subscribe(Subscriber<? super ByteBuffer> subscriber) {
+            if (stream != null) {
+                stream.connectionComplete = true;
+                if (stream.pout != null) {
+                    synchronized (stream.pout) {
+                        stream.pout.notifyAll();
+                    }
+
+                    BodyPublishers.ofInputStream(pin).subscribe(subscriber);
+                    stream = null;
+                    pin = null;
+                    return;
+                }
+            }
+            BodyPublishers.noBody().subscribe(subscriber);
+        }
+
+        @Override
+        public long contentLength() {
+            if (stream != null && stream.pout != null) {
+                final CloseableByteArrayOutputStream baos = (CloseableByteArrayOutputStream) stream.pout;
+
+                try {
+                    synchronized (baos) {
+                        if (!baos.closed) {
+                            baos.wait(csPolicy.getConnectionTimeout());
+                        }
+                    }
+                    contentLen = (int) baos.size();
+                } catch (InterruptedException e) {
+                    //ignore
+                }
+            }
+
+            return contentLen;
+        }
+    }
+
+    /**
+     * The {@link ByteArrayOutputStream} implementation that tracks the closeability state.
+     */
+    private static final class CloseableByteArrayOutputStream extends ByteArrayOutputStream {
+        private boolean closed;
+
+        /**
+         * Creates a new output stream for user data
+         */
+        CloseableByteArrayOutputStream() {
+            super(4096);
+        }
+
+        /**
+         * Writes the specified byte to this output stream.
+         *
+         * @param   b   the byte to be written.
+         */
+        public synchronized void write(int b) {
+            if (closed) {
+                return;
+            }
+            super.write(b);
+        }
+
+        /**
+         * Writes <code>len</code> bytes from the specified byte array
+         * starting at offset <code>off</code> to this output stream.
+         *
+         * @param   b     the data.
+         * @param   off   the start offset in the data.
+         * @param   len   the number of bytes to write.
+         */
+        public synchronized void write(byte[] b, int off, int len) {
+            if (closed) {
+                return;
+            }
+            super.write(b, off, len);
+        }
+
+        /**
+         * Resets the <code>count</code> field of this output
+         * stream to zero, so that all currently accumulated output in the
+         * output stream is discarded. The output stream can be used again,
+         * reusing the already allocated buffer space. If the output stream
+         * has been closed, then this method has no effect.
+         *
+         * @see     java.io.ByteArrayInputStream#count
+         */
+        public synchronized void reset() {
+            if (closed) {
+                return;
+            }
+            super.reset();
+        }
+
+        /**
+         * After close() has been called, it is no longer possible to write
+         * to this stream. Further calls to write will have no effect.
+         */
+        public synchronized void close() throws IOException {
+            closed = true;
+            super.close();
+            notifyAll();
+        }
+
+        /**
+         * Returns new instance of the {@link ByteArrayInputStream} that uses the same underlying buffer as 
+         * this stream. The steam must be closed in order to ensure no further modifications could happen. 
+         * @return new instance of the {@link ByteArrayInputStream}
+         */
+        public ByteArrayInputStream getInputStream() {
+            if (!closed) {
+                throw new IllegalStateException("The stream is not closed and underlying buffer "
+                        + "could still be changed");
+            }
+
+            // Creates new ByteArrayInputStream instance that respects the current state of the buffer 
+            // (since ByteArrayInputStream::toByteArray() does array copy).
+            return new ByteArrayInputStream(this.buf, 0, this.count);
+        }
+    }
+
+    class HttpClientWrappedOutputStream extends WrappedOutputStream {  
 
         List<Flow.Subscriber<? super ByteBuffer>> subscribers = new LinkedList<>();
         CompletableFuture<HttpResponse<InputStream>> future;
@@ -655,8 +807,8 @@ public class HttpClientHTTPConduit extends URLConnectionHTTPConduit {
         int rtimeout;
         volatile Throwable exception;
         volatile boolean connectionComplete;
-        PipedOutputStream pout;
-        HttpClientBodyPublisher publisher;
+        OutputStream pout;
+        CloseableBodyPublisher publisher;
         HttpRequest request;
 
 
@@ -801,12 +953,20 @@ public class HttpClientHTTPConduit extends URLConnectionHTTPConduit {
                 contentLen = 0;
             }
 
-            final PipedInputStream pin = new PipedInputStream(csPolicy.getChunkLength() <= 0
-                ? 4096 : csPolicy.getChunkLength());
-
-            this.publisher = new HttpClientBodyPublisher(this, pin);
-            if (contentLen != 0) {
-                pout = new HttpClientPipedOutputStream(this, pin, csPolicy, publisher);
+            if (csPolicy.isAllowChunking() || contentLen >= 0) {
+                final PipedInputStream pin = new PipedInputStream(csPolicy.getChunkLength() <= 0
+                        ? 4096 : csPolicy.getChunkLength());
+                this.publisher = new HttpClientBodyPublisher(this, () -> pin);
+                if (contentLen != 0) {
+                    pout = new HttpClientPipedOutputStream(this, pin, csPolicy, publisher);
+                }
+            } else if (contentLen != 0) {
+                // If chunking is not allowed but the contentLen is unknown (-1), we need to
+                // buffer the request body stream until it is fully flushed by the client and only
+                // than send the request.
+                final CloseableByteArrayOutputStream baos = new CloseableByteArrayOutputStream();
+                this.publisher = new HttpClientSizedBodyPublisher(this, csPolicy, baos::getInputStream);
+                pout = baos;
             }
 
             HttpRequest.Builder rb = HttpRequest.newBuilder()
diff --git a/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/FileStore.java b/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/FileStore.java
new file mode 100644
index 0000000000..673c91ad6b
--- /dev/null
+++ b/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/FileStore.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cxf.systest.jaxrs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.activation.DataHandler;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.container.AsyncResponse;
+import javax.ws.rs.container.Suspended;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import javax.ws.rs.core.StreamingOutput;
+import javax.ws.rs.core.UriInfo;
+
+import org.apache.cxf.common.util.StringUtils;
+import org.apache.cxf.helpers.IOUtils;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
+
+@Path("/file-store")
+public class FileStore {
+    private final ConcurrentMap<String, byte[]> store = new ConcurrentHashMap<>();
+    @Context private HttpHeaders headers;
+
+    @POST
+    @Path("/stream")
+    @Consumes("*/*")
+    public Response addBook(@QueryParam("chunked") boolean chunked, InputStream in) throws IOException {
+        String transferEncoding = headers.getHeaderString("Transfer-Encoding");
+
+        if (chunked != Objects.equals("chunked", transferEncoding)) {
+            throw new WebApplicationException(Status.EXPECTATION_FAILED);
+        }
+
+        try (in) {
+            if (chunked) {
+                return Response.ok(new StreamingOutput() {
+                    @Override
+                    public void write(OutputStream out) throws IOException, WebApplicationException {
+                        in.transferTo(out);
+                    }
+                }).build();
+            } else {
+                // Make sure we have small amount of data for chunking to not kick in
+                final byte[] content = in.readAllBytes(); 
+                return Response.ok(Arrays.copyOf(content, content.length / 10)).build();
+            }
+        }
+    }
+
+    @POST
+    @Consumes("multipart/form-data")
+    public void addBook(@QueryParam("chunked") boolean chunked, 
+            @Suspended final AsyncResponse response, @Context final UriInfo uri, final MultipartBody body)  {
+        
+        String transferEncoding = headers.getHeaderString("Transfer-Encoding");
+        if (chunked != Objects.equals("chunked", transferEncoding)) {
+            response.resume(Response.status(Status.EXPECTATION_FAILED).build());
+            return;
+        }
+
+        for (final Attachment attachment: body.getAllAttachments()) {
+            final DataHandler handler = attachment.getDataHandler();
+
+            if (handler != null) {
+                final String source = handler.getName();
+                if (StringUtils.isEmpty(source)) {
+                    response.resume(Response.status(Status.BAD_REQUEST).build());
+                    return;
+                }
+
+                try {
+                    if (store.containsKey(source)) {
+                        response.resume(Response.status(Status.CONFLICT).build());
+                        return;
+                    }
+
+                    final byte[] content = IOUtils.readBytesFromStream(handler.getInputStream());
+                    if (store.putIfAbsent(source, content) != null) {
+                        response.resume(Response.status(Status.CONFLICT).build());
+                        return;
+                    }
+                } catch (final Exception ex) {
+                    response.resume(Response.serverError().build());
+                }
+
+                if (response.isSuspended()) {
+                    response.resume(Response.created(uri.getRequestUriBuilder().path(source).build())
+                            .build());
+                }
+            }
+        }
+
+        if (response.isSuspended()) {
+            response.resume(Response.status(Status.BAD_REQUEST).build());
+        }
+    }
+}
diff --git a/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/FileStoreServer.java b/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/FileStoreServer.java
new file mode 100644
index 0000000000..34886256b5
--- /dev/null
+++ b/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/FileStoreServer.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cxf.systest.jaxrs;
+
+import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
+import org.apache.cxf.jaxrs.lifecycle.SingletonResourceProvider;
+import org.apache.cxf.testutil.common.AbstractBusTestServerBase;
+
+class FileStoreServer extends AbstractBusTestServerBase {
+    private org.apache.cxf.endpoint.Server server;
+    private final String port;
+
+    FileStoreServer(String port) {
+        this.port = port;
+    }
+
+    protected void run() {
+        JAXRSServerFactoryBean sf = new JAXRSServerFactoryBean();
+        sf.setResourceClasses(FileStore.class);
+        sf.setResourceProvider(FileStore.class, new SingletonResourceProvider(new FileStore(), true));
+        sf.setAddress("http://localhost:" + port);
+        server = sf.create();
+    }
+
+    public void tearDown() throws Exception {
+        server.stop();
+        server.destroy();
+        server = null;
+    }
+}
\ No newline at end of file
diff --git a/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/JAXRSClientChunkingTest.java b/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/JAXRSClientChunkingTest.java
new file mode 100644
index 0000000000..536b4aecb3
--- /dev/null
+++ b/systests/transports/src/test/java/org/apache/cxf/systest/jaxrs/JAXRSClientChunkingTest.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cxf.systest.jaxrs;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
+import org.apache.cxf.jaxrs.impl.MetadataMap;
+import org.apache.cxf.jaxrs.model.AbstractResourceInfo;
+import org.apache.cxf.jaxrs.provider.MultipartProvider;
+import org.apache.cxf.testutil.common.AbstractBusClientServerTestBase;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized.Parameters;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = org.junit.runners.Parameterized.class)
+public class JAXRSClientChunkingTest extends AbstractBusClientServerTestBase {
+    private static final String PORT = allocatePort(FileStoreServer.class);
+    private final Boolean chunked;
+
+    public JAXRSClientChunkingTest(Boolean chunked) {
+        this.chunked = chunked;
+    }
+
+    @BeforeClass
+    public static void startServers() throws Exception {
+        AbstractResourceInfo.clearAllMaps();
+        assertTrue("server did not launch correctly", launchServer(new FileStoreServer(PORT)));
+        createStaticBus();
+    }
+    
+    @Parameters(name = "{0}")
+    public static Collection<Boolean> data() {
+        return Arrays.asList(new Boolean[] {Boolean.FALSE, Boolean.TRUE});
+    }
+
+    @Test
+    public void testMultipartChunking() {
+        final String url = "http://localhost:" + PORT + "/file-store";
+        final WebClient webClient = WebClient.create(url, List.of(new MultipartProvider())).query("chunked", chunked);
+        WebClient.getConfig(webClient).getHttpConduit().getClient().setAllowChunking(chunked);
+
+        try {
+            final String filename = "keymanagers.jks";
+            final MultivaluedMap<String, String> headers = new MetadataMap<>();
+            headers.add("Content-ID", filename);
+            headers.add("Content-Type", "application/binary");
+            headers.add("Content-Disposition", "attachment; filename=" + chunked + "_" + filename);
+            final Attachment att = new Attachment(getClass().getResourceAsStream("/" + filename), headers);
+            final MultipartBody entity = new MultipartBody(att);
+            try (Response response = webClient.header("Content-Type", "multipart/form-data").post(entity)) {
+                assertThat(response.getStatus(), equalTo(201));
+            }
+        } finally {
+            webClient.close();
+        }
+    }
+    
+    @Test
+    public void testStreamChunking() throws IOException {
+        final String url = "http://localhost:" + PORT + "/file-store/stream";
+        final WebClient webClient = WebClient.create(url).query("chunked", chunked);
+        WebClient.getConfig(webClient).getHttpConduit().getClient().setAllowChunking(chunked);
+
+        final byte[] bytes = new byte [32 * 1024];
+        final Random random = new Random();
+        random.nextBytes(bytes);
+
+        try (InputStream in = new ByteArrayInputStream(bytes)) {
+            final Entity<InputStream> entity = Entity.entity(in, MediaType.APPLICATION_OCTET_STREAM);
+            try (Response response = webClient.post(entity)) {
+                assertThat(response.getStatus(), equalTo(200));
+                assertThat(response.getHeaderString("Transfer-Encoding"), equalTo(chunked ? "chunked" : null));
+                assertThat(response.getEntity(), not(equalTo(null)));
+            }
+        } finally {
+            webClient.close();
+        }
+    }
+}