[pinot] branch master updated: use storage.createFrom instead of copies in for-loop by hand (#8918)

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 52bf09dcf7 use storage.createFrom instead of copies in for-loop by hand (#8918)
52bf09dcf7 is described below

commit 52bf09dcf7dd6cb250710dad59133fb32ce63916
Author: Xiaobing <61...@users.noreply.github.com>
AuthorDate: Fri Jun 17 17:20:15 2022 -0700

    use storage.createFrom instead of copies in for-loop by hand (#8918)
---
 .../apache/pinot/plugin/filesystem/GcsPinotFS.java    | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
index 62eac87894..dc5a1787e2 100644
--- a/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
+++ b/pinot-plugins/pinot-file-system/pinot-gcs/src/main/java/org/apache/pinot/plugin/filesystem/GcsPinotFS.java
@@ -21,8 +21,9 @@ package org.apache.pinot.plugin.filesystem;
 import com.google.api.gax.paging.Page;
 import com.google.auth.Credentials;
 import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.WriteChannel;
 import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
 import com.google.cloud.storage.Bucket;
 import com.google.cloud.storage.CopyWriter;
 import com.google.cloud.storage.Storage;
@@ -37,9 +38,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.net.URI;
-import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
-import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
@@ -58,7 +57,6 @@ public class GcsPinotFS extends BasePinotFS {
   public static final String GCP_KEY = "gcpKey";
 
   private static final Logger LOGGER = LoggerFactory.getLogger(GcsPinotFS.class);
-  private static final int BUFFER_SIZE = 128 * 1024;
   // See https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
   private static final int BATCH_LIMIT = 100;
   private Storage _storage;
@@ -181,17 +179,8 @@ public class GcsPinotFS extends BasePinotFS {
     LOGGER.info("Copying file {} to uri {}", srcFile.getAbsolutePath(), dstUri);
     GcsUri dstGcsUri = new GcsUri(dstUri);
     checkState(!isPathTerminatedByDelimiter(dstGcsUri), "Path '%s' must be a filename", dstGcsUri);
-    Blob blob = getBucket(dstGcsUri).create(dstGcsUri.getPath(), new byte[0]);
-    WriteChannel writeChannel = blob.writer();
-    writeChannel.setChunkSize(BUFFER_SIZE);
-    ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
-    SeekableByteChannel channel = Files.newByteChannel(srcFile.toPath());
-    for (int bytesRead = channel.read(buffer); bytesRead != -1; bytesRead = channel.read(buffer)) {
-      buffer.flip();
-      writeChannel.write(buffer);
-      buffer.clear();
-    }
-    writeChannel.close();
+    BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(dstGcsUri.getBucketName(), dstGcsUri.getPath())).build();
+    _storage.createFrom(blobInfo, Files.newInputStream(srcFile.toPath()));
   }
 
   @Override


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org