[iotdb] branch tiered_storage updated: add interface size() for objectStorage

[xi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xingtanzjr pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/tiered_storage by this push:
     new ece6708674c add interface size() for objectStorage
ece6708674c is described below

commit ece6708674ce569e265a443dbd4d60b34d797f4d
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Thu May 11 15:06:04 2023 +0800

    add interface size() for objectStorage
---
 .../java/org/apache/iotdb/os/fileSystem/OSFile.java   | 18 +++++++++---------
 .../apache/iotdb/os/io/ObjectStorageConnector.java    |  5 +++--
 .../iotdb/os/io/aws/S3ObjectStorageConnector.java     | 19 +++++++++++++++++--
 3 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSFile.java b/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSFile.java
index 722500dae26..8012ee3808c 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSFile.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/fileSystem/OSFile.java
@@ -18,12 +18,6 @@
  */
 package org.apache.iotdb.os.fileSystem;
 
-import java.io.FileFilter;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.file.Path;
 import org.apache.iotdb.os.conf.ObjectStorageConfig;
 import org.apache.iotdb.os.conf.ObjectStorageDescriptor;
 import org.apache.iotdb.os.exception.ObjectStorageException;
@@ -34,14 +28,20 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.FileFilter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
 import java.net.URI;
+import java.net.URL;
+import java.nio.file.Path;
 
 import static org.apache.iotdb.os.utils.ObjectStorageConstant.FILE_SEPARATOR;
 
-
 public class OSFile extends File {
   private static final Logger logger = LoggerFactory.getLogger(OSFile.class);
-  private static final String UNSUPPORT_OPERATION = "Current object storage file doesn't support this operation.";
+  private static final String UNSUPPORT_OPERATION =
+      "Current object storage file doesn't support this operation.";
   private static final ObjectStorageConfig config =
       ObjectStorageDescriptor.getInstance().getConfig();
   private static final ObjectStorageConnector connector;
@@ -97,7 +97,7 @@ public class OSFile extends File {
   @Override
   public File getParentFile() {
     int lastSeparatorIdx = osUri.getKey().lastIndexOf(FILE_SEPARATOR);
-    if(lastSeparatorIdx <= 0) {
+    if (lastSeparatorIdx <= 0) {
       return null;
     }
     return new OSFile(new OSURI(osUri.getBucket(), osUri.getKey().substring(0, lastSeparatorIdx)));
diff --git a/object-storage/src/main/java/org/apache/iotdb/os/io/ObjectStorageConnector.java b/object-storage/src/main/java/org/apache/iotdb/os/io/ObjectStorageConnector.java
index 2e70268bdcd..154417a4696 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/io/ObjectStorageConnector.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/io/ObjectStorageConnector.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.os.fileSystem.OSURI;
 public interface ObjectStorageConnector {
   boolean doesObjectExist(OSURI osuri) throws ObjectStorageException;
 
-  void write(String sourceFile, String containerName, String targetFileName)
-      throws ObjectStorageException;
+  long size(String fileName) throws ObjectStorageException;
+
+  void write(String sourceFile, String targetFileName) throws ObjectStorageException;
 }
diff --git a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java b/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java
index 31cd148bcc0..f294daf82ac 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java
@@ -27,11 +27,15 @@ import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
 
 import java.io.File;
+import java.util.List;
 
 public class S3ObjectStorageConnector implements ObjectStorageConnector {
   private S3Client s3Client;
@@ -58,9 +62,20 @@ public class S3ObjectStorageConnector implements ObjectStorageConnector {
     return true;
   }
 
+  public long size(String fileName) throws ObjectStorageException {
+    ListObjectsV2Request listObjectsRequest =
+        ListObjectsV2Request.builder().bucket(AWSS3Config.getBucketName()).prefix(fileName).build();
+    ListObjectsV2Response res = s3Client.listObjectsV2(listObjectsRequest);
+    List<S3Object> objects = res.contents();
+    if (objects.size() != 1) {
+      throw new ObjectStorageException(
+          String.format("expected 1 S3Object with prefix %s but get %d", fileName, objects.size()));
+    }
+    return objects.get(0).size();
+  }
+
   @Override
-  public void write(String sourceFile, String containerName, String targetFileName)
-      throws ObjectStorageException {
+  public void write(String sourceFile, String targetFileName) throws ObjectStorageException {
     try {
       PutObjectRequest putOb =
           PutObjectRequest.builder()