You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2022/02/24 18:06:44 UTC
[camel] branch camel-3.11.x updated: CAMEL-17702: fix NPE and improve handling for Content-Length (#7033)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch camel-3.11.x
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/camel-3.11.x by this push:
new 639f574 CAMEL-17702: fix NPE and improve handling for Content-Length (#7033)
639f574 is described below
commit 639f574163f94bab0519da2b1cb3a5e9fa357daa
Author: klease <38...@users.noreply.github.com>
AuthorDate: Thu Feb 24 19:02:14 2022 +0100
CAMEL-17702: fix NPE and improve handling for Content-Length (#7033)
Refactor and simplify code for calculating content-length to
avoid NPE and unnecessary work.
Create the Blob directly from InputStream.
Add a unit test for these cases.
---
.../google/storage/GoogleCloudStorageProducer.java | 65 ++++++------
.../google/storage/unit/ProducerStoreFileTest.java | 118 +++++++++++++++++++++
.../src/test/resources/tostore/FileToStore.txt | 1 +
3 files changed, 149 insertions(+), 35 deletions(-)
diff --git a/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageProducer.java b/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageProducer.java
index 6f479a8..a678cca 100644
--- a/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageProducer.java
+++ b/components/camel-google/camel-google-storage/src/main/java/org/apache/camel/component/google/storage/GoogleCloudStorageProducer.java
@@ -23,14 +23,12 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
-import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import com.google.cloud.WriteChannel;
import com.google.cloud.storage.Blob;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.BlobInfo;
@@ -101,9 +99,7 @@ public class GoogleCloudStorageProducer extends DefaultProducer {
Map<String, String> objectMetadata = determineMetadata(exchange);
- File filePayload;
InputStream is;
- ByteArrayOutputStream baos = null;
Object obj = exchange.getIn().getMandatoryBody();
// Need to check if the message body is WrappedFile
@@ -111,52 +107,48 @@ public class GoogleCloudStorageProducer extends DefaultProducer {
obj = ((WrappedFile<?>) obj).getFile();
}
if (obj instanceof File) {
- filePayload = (File) obj;
+ File filePayload = (File) obj;
is = new FileInputStream(filePayload);
} else {
is = exchange.getIn().getMandatoryBody(InputStream.class);
- baos = determineLengthInputStream(is);
-
- if (objectMetadata.containsKey(Exchange.CONTENT_LENGTH)) {
- if (objectMetadata.get("Content-Length").equals("0")
- && ObjectHelper.isEmpty(exchange.getProperty(Exchange.CONTENT_LENGTH))) {
- LOG.debug(
- "The content length is not defined. It needs to be determined by reading the data into memory");
- objectMetadata.put("Content-Length", String.valueOf(baos.size()));
- is = new ByteArrayInputStream(baos.toByteArray());
- } else {
- if (ObjectHelper.isNotEmpty(exchange.getProperty(Exchange.CONTENT_LENGTH))) {
- objectMetadata.put("Content-Length",
- exchange.getProperty(Exchange.CONTENT_LENGTH, String.class));
- }
- }
- }
}
+ // Handle Content-Length if not already set
+ is = setContentLength(objectMetadata, is);
Blob createdBlob;
BlobId blobId = BlobId.of(bucketName, objectName);
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setMetadata(objectMetadata).build();
- if (baos.size() > 1_000_000) {
- // When content is not available or large (1MB or more) it is recommended
- // to write it in chunks via the blob's channel writer.
- try (WriteChannel writer = storage.writer(blobInfo)) {
- writer.write(ByteBuffer.wrap(baos.toByteArray()));
- }
- createdBlob = storage.get(blobId);
- } else {
- byte[] bytes = baos.toByteArray();
- // create the blob in one request.
- createdBlob = storage.create(blobInfo, bytes);
- }
-
+ // According to documentation, this internally uses a WriteChannel
+ createdBlob = storage.createFrom(blobInfo, is);
LOG.trace("created createdBlob [{}]", createdBlob);
Message message = getMessageForResponse(exchange);
message.setBody(createdBlob);
- IOHelper.close(baos);
IOHelper.close(is);
}
+ /**
+ * If no content-length header was found, calculate length by reading the content.
+ *
+ * @param objectMetadata Metadata set from Exchange headers
+ * @param is InputStream to read the Exchange body content
+ * @return the original InputStream if Content-Length is set or a ByteArrayInputStream if the
+ * original stream was read to determine the length.
+ * @throws IOException if the InputStream cannot be read.
+ */
+ private InputStream setContentLength(Map<String, String> objectMetadata, InputStream is) throws IOException {
+ if (!objectMetadata.containsKey(Exchange.CONTENT_LENGTH) ||
+ objectMetadata.get(Exchange.CONTENT_LENGTH).equals("0")) {
+ LOG.debug(
+ "The content length is not defined. It needs to be determined by reading the data into memory");
+ ByteArrayOutputStream baos = determineLengthInputStream(is);
+ objectMetadata.put("Content-Length", String.valueOf(baos.size()));
+ return new ByteArrayInputStream(baos.toByteArray());
+ } else {
+ return is;
+ }
+ }
+
private ByteArrayOutputStream determineLengthInputStream(InputStream is) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] bytes = new byte[1024];
@@ -173,6 +165,9 @@ public class GoogleCloudStorageProducer extends DefaultProducer {
Long contentLength = exchange.getIn().getHeader(GoogleCloudStorageConstants.CONTENT_LENGTH, Long.class);
if (contentLength != null) {
objectMetadata.put("Content-Length", String.valueOf(contentLength));
+ } else if (ObjectHelper.isNotEmpty(exchange.getProperty(Exchange.CONTENT_LENGTH))) {
+ objectMetadata.put("Content-Length",
+ exchange.getProperty(Exchange.CONTENT_LENGTH, String.class));
}
String contentType = exchange.getIn().getHeader(GoogleCloudStorageConstants.CONTENT_TYPE, String.class);
diff --git a/components/camel-google/camel-google-storage/src/test/java/org/apache/camel/component/google/storage/unit/ProducerStoreFileTest.java b/components/camel-google/camel-google-storage/src/test/java/org/apache/camel/component/google/storage/unit/ProducerStoreFileTest.java
new file mode 100644
index 0000000..fdda3cf
--- /dev/null
+++ b/components/camel-google/camel-google-storage/src/test/java/org/apache/camel/component/google/storage/unit/ProducerStoreFileTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.google.storage.unit;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import com.google.cloud.storage.Blob;
+import org.apache.camel.EndpointInject;
+import org.apache.camel.Exchange;
+import org.apache.camel.ProducerTemplate;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.file.FileConsumer;
+import org.apache.camel.component.file.GenericFile;
+import org.apache.camel.component.google.storage.GoogleCloudStorageConstants;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ProducerStoreFileTest extends GoogleCloudStorageBaseTest {
+ private static final String FILE_ENDPOINT = "target/test-classes/tostore";
+ private static final String FILE_NAME = "FileToStore.txt";
+
+ @EndpointInject
+ private ProducerTemplate template;
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ public void configure() {
+ String endpoint = "google-storage://myCamelBucket?autoCreateBucket=true";
+
+ from("direct:fromFile").to(endpoint);
+ }
+ };
+ }
+
+ @Test
+ public void testStoreFromFileWithContentLength() throws InterruptedException, IOException {
+ Path filePath = Path.of(FILE_ENDPOINT, FILE_NAME);
+ Exchange storeFileExchange = template.request("direct:fromFile", exchange -> {
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.OBJECT_NAME, FILE_NAME);
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.CONTENT_ENCODING, "text/plain");
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.CONTENT_LENGTH, Files.size(filePath));
+ GenericFile<File> genericFile = FileConsumer.asGenericFile(FILE_ENDPOINT,
+ filePath.toFile(), StandardCharsets.UTF_8.name(), false);
+ exchange.getIn().setBody(genericFile);
+ });
+ assertNotNull(storeFileExchange);
+ Blob fileBlob = storeFileExchange.getMessage().getBody(Blob.class);
+ assertNotNull(fileBlob);
+ assertEquals(FILE_NAME, fileBlob.getName());
+ assertEquals(Files.size(filePath), Long.valueOf(fileBlob.getMetadata().get("Content-Length")));
+ byte[] blobContents = fileBlob.getContent();
+ assertNotNull(blobContents);
+ assertEquals(Files.readString(filePath), new String(blobContents));
+ }
+
+ @Test
+ public void testStoreFromFileWithNoContentLength() throws InterruptedException, IOException {
+ Path filePath = Path.of(FILE_ENDPOINT, FILE_NAME);
+ Exchange storeFileExchange = template.request("direct:fromFile", exchange -> {
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.OBJECT_NAME, FILE_NAME);
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.CONTENT_ENCODING, "text/plain");
+ GenericFile<File> genericFile = FileConsumer.asGenericFile(FILE_ENDPOINT,
+ filePath.toFile(), StandardCharsets.UTF_8.name(), false);
+ exchange.getIn().setBody(genericFile);
+ });
+ assertNotNull(storeFileExchange);
+ Blob fileBlob = storeFileExchange.getMessage().getBody(Blob.class);
+ assertNotNull(fileBlob);
+ assertEquals(FILE_NAME, fileBlob.getName());
+ assertEquals(Files.size(filePath), Long.valueOf(fileBlob.getMetadata().get("Content-Length")));
+ byte[] blobContents = fileBlob.getContent();
+ assertNotNull(blobContents);
+ assertEquals(Files.readString(filePath), new String(blobContents));
+ }
+
+ @Test
+ public void testStoreFromstream() throws InterruptedException {
+ final String fileName = "FromStream.txt";
+ byte[] payload = "Hi, How are you ?".getBytes();
+ ByteArrayInputStream bais = new ByteArrayInputStream(payload);
+ Exchange storeFileExchange = template.request("direct:fromFile", exchange -> {
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.OBJECT_NAME, fileName);
+ exchange.getIn().setHeader(GoogleCloudStorageConstants.CONTENT_ENCODING, "text/plain");
+ exchange.getIn().setBody(bais);
+ });
+ assertNotNull(storeFileExchange);
+ Blob fileBlob = storeFileExchange.getMessage().getBody(Blob.class);
+ assertNotNull(fileBlob);
+ assertEquals(fileName, fileBlob.getName());
+ assertEquals(payload.length, Integer.valueOf(fileBlob.getMetadata().get("Content-Length")));
+ byte[] blobContents = fileBlob.getContent();
+ assertTrue(Arrays.equals(payload, blobContents));
+ }
+}
diff --git a/components/camel-google/camel-google-storage/src/test/resources/tostore/FileToStore.txt b/components/camel-google/camel-google-storage/src/test/resources/tostore/FileToStore.txt
new file mode 100644
index 0000000..184362b
--- /dev/null
+++ b/components/camel-google/camel-google-storage/src/test/resources/tostore/FileToStore.txt
@@ -0,0 +1 @@
+This is the file to store on google cloud storage.
\ No newline at end of file