You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2021/10/20 16:45:32 UTC
[lucene-solr] branch branch_8x updated: SOLR-15702: Stabilize S3
tests by using waiters (#356)
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 1cc9f4d SOLR-15702: Stabilize S3 tests by using waiters (#356)
1cc9f4d is described below
commit 1cc9f4d4dcdfb5733baeb0a237e73a41becfe4d9
Author: Houston Putman <ho...@apache.org>
AuthorDate: Wed Oct 20 11:25:31 2021 -0400
SOLR-15702: Stabilize S3 tests by using waiters (#356)
---
solr/CHANGES.txt | 3 ++-
.../java/org/apache/solr/s3/S3StorageClient.java | 25 ++++++++++++++--------
2 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4c077e5..2a05155 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -17,7 +17,8 @@ New Features
Improvements
---------------------
-(No changes)
+
+* SOLR-15702: Stabilize S3 tests by using waiters after directory creation (Houston Putman)
Optimizations
---------------------
diff --git a/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java b/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java
index 04a385e..31ac598 100644
--- a/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java
+++ b/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.net.URI;
+import java.time.Duration;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
@@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.retry.backoff.BackoffStrategy;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
@@ -130,10 +132,10 @@ public class S3StorageClient {
/** Create a directory in S3. */
void createDirectory(String path) throws S3Exception {
- path = sanitizedDirPath(path);
+ String sanitizedDirPath = sanitizedDirPath(path);
- if (!parentDirectoryExist(path)) {
- createDirectory(getParentDirectory(path));
+ if (!parentDirectoryExist(sanitizedDirPath)) {
+ createDirectory(getParentDirectory(sanitizedDirPath));
// TODO see https://issues.apache.org/jira/browse/SOLR-15359
// throw new S3Exception("Parent directory doesn't exist, path=" + path);
}
@@ -144,9 +146,18 @@ public class S3StorageClient {
PutObjectRequest.builder()
.bucket(bucketName)
.contentType(S3_DIR_CONTENT_TYPE)
- .key(path)
+ .key(sanitizedDirPath)
.build();
s3Client.putObject(putRequest, RequestBody.empty());
+ // Wait until the object exists to continue
+ s3Client
+ .waiter()
+ .waitUntilObjectExists(
+ builder -> builder.bucket(bucketName).key(sanitizedDirPath),
+ config ->
+ config
+ .waitTimeout(Duration.ofSeconds(5))
+ .backoffStrategy(BackoffStrategy.defaultStrategy()));
} catch (SdkClientException ase) {
throw handleAmazonException(ase);
}
@@ -447,11 +458,7 @@ public class S3StorageClient {
}
// Check for existence twice, because s3Mock has issues in the tests
- if (pathExists(parentDirectory)) {
- return true;
- } else {
- return pathExists(parentDirectory);
- }
+ return pathExists(parentDirectory);
}
private String getParentDirectory(String path) {