You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2023/03/18 02:54:37 UTC
[asterixdb] 03/11: [NO ISSUE][EXT]: Properly handle failures and retries for GCS
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5f2c24f4db895fe6d91f90336ab2502783b2a8f7
Author: Hussain Towaileb <Hu...@couchbase.com>
AuthorDate: Mon Mar 6 21:41:16 2023 +0300
[NO ISSUE][EXT]: Properly handle failures and retries for GCS
Change-Id: I0bd42c391db603040d0470e976fa561649024992
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17409
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Wail Alkowaileet <wa...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../input/record/reader/gcs/GCSInputStream.java | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
index 652fa3e4b0..5da4583935 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
@@ -47,7 +47,7 @@ public class GCSInputStream extends AbstractExternalInputStream {
private final Storage client;
private final String container;
- private static final int MAX_RETRIES = 5; // We will retry 5 times in case of retryable errors
+ private static final int MAX_ATTEMPTS = 5; // We try a total of 5 times in case of retryable errors
public GCSInputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException {
super(configuration, filePaths);
@@ -78,10 +78,10 @@ public class GCSInputStream extends AbstractExternalInputStream {
* @return true
*/
private boolean doGetInputStream(String fileName) throws RuntimeDataException {
- int retries = 0;
+ int attempt = 0;
BlobId blobId = BlobId.of(container, fileName);
- while (retries < MAX_RETRIES) {
+ while (attempt < MAX_ATTEMPTS) {
try {
Blob blob = client.get(blobId);
if (blob == null) {
@@ -93,14 +93,14 @@ public class GCSInputStream extends AbstractExternalInputStream {
in = new ByteArrayInputStream(blob.getContent());
break;
} catch (BaseServiceException ex) {
- if (!shouldRetry(retries++) && ex.isRetryable()) {
+ if (!ex.isRetryable() || !shouldRetry(++attempt)) {
throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
}
- LOGGER.debug(() -> "Retryable error: " + LogRedactionUtil.userData(ex.getMessage()));
+ LOGGER.debug(() -> "Retryable error: " + getMessageOrToString(ex));
- // Backoff for 1 sec for the first 2 retries, and 2 seconds from there onward
+ // Backoff for 1 sec for the first 3 attempts, and 2 seconds from there onward
try {
- Thread.sleep(TimeUnit.SECONDS.toMillis(retries < 3 ? 1 : 2));
+ Thread.sleep(TimeUnit.SECONDS.toMillis(attempt < 3 ? 1 : 2));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -111,8 +111,8 @@ public class GCSInputStream extends AbstractExternalInputStream {
return true;
}
- private boolean shouldRetry(int currentRetry) {
- return currentRetry < MAX_RETRIES;
+ private boolean shouldRetry(int nextAttempt) {
+ return nextAttempt < MAX_ATTEMPTS;
}
@Override