You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by bh...@apache.org on 2015/03/04 12:30:11 UTC
[4/5] git commit: updated refs/heads/4.5 to 1f97f94
listDirectory method updated to use ObjectListing.isTruncated().
Because buckets can contain a virtually unlimited number of keys, the
complete results of a list query can be extremely large. To manage large
result sets, Amazon S3 uses pagination to split them into multiple
responses.
Signed-off-by: Rajani Karuturi <ra...@gmail.com>
This closes #25
(cherry picked from commit 0b6c540a203453580bb76a0d0a60be292fccfa0e)
Signed-off-by: Rohit Yadav <ro...@shapeblue.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/0ae5912f
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/0ae5912f
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/0ae5912f
Branch: refs/heads/4.5
Commit: 0ae5912f0019b9ea4c496ea138ecb57fc840ef23
Parents: 8069b17
Author: santhosh <sa...@47line.com>
Authored: Fri Oct 24 21:15:29 2014 +0530
Committer: Rohit Yadav <ro...@shapeblue.com>
Committed: Wed Mar 4 16:59:12 2015 +0530
----------------------------------------------------------------------
utils/src/com/cloud/utils/S3Utils.java | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0ae5912f/utils/src/com/cloud/utils/S3Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java
index d9b850d..37aeb91 100644
--- a/utils/src/com/cloud/utils/S3Utils.java
+++ b/utils/src/com/cloud/utils/S3Utils.java
@@ -57,13 +57,14 @@ import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
-
import com.cloud.utils.exception.CloudRuntimeException;
public final class S3Utils {
@@ -338,9 +339,19 @@ public final class S3Utils {
private static List<S3ObjectSummary> listDirectory(final String bucketName, final String directory, final AmazonS3 client) {
- final List<S3ObjectSummary> objects = client.listObjects(bucketName, directory + SEPARATOR).getObjectSummaries();
-
- if (objects == null) {
+ List<S3ObjectSummary> objects = new ArrayList<S3ObjectSummary>();
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(directory + SEPARATOR);
+ ObjectListing objectListing;
+
+ do {
+ objectListing = client.listObjects(listObjectsRequest);
+
+ if (objectListing != null )
+ objects.addAll(objectListing.getObjectSummaries());
+ listObjectsRequest.setMarker(objectListing.getNextMarker());
+ } while (objectListing.isTruncated());
+
+ if (objects.isEmpty()) {
return emptyList();
}