You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2023/03/09 12:13:51 UTC
[hadoop] branch branch-3.3.5 updated: HADOOP-17836. Improve logging on ABFS error reporting (#3281)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3.5 by this push:
new 419e3e3ec5c HADOOP-17836. Improve logging on ABFS error reporting (#3281)
419e3e3ec5c is described below
commit 419e3e3ec5c190ee4fd468a33c2c3e2631054b4c
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Wed Aug 18 11:39:17 2021 +0100
HADOOP-17836. Improve logging on ABFS error reporting (#3281)
Contributed by Steve Loughran.
---
.../contracts/exceptions/InvalidAbfsRestOperationException.java | 7 +++++--
.../org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java | 4 +++-
.../org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java | 6 +++---
.../org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java | 4 ++--
.../org/apache/hadoop/fs/azurebfs/services/ReadBufferWorker.java | 6 +++++-
5 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java
index aba1d8c1efa..19620212134 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java
@@ -24,7 +24,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
/**
- * Exception to wrap invalid Azure service error responses.
+ * Exception to wrap invalid Azure service error responses and exceptions
+ * raised on network IO.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@ -34,7 +35,9 @@ public class InvalidAbfsRestOperationException extends AbfsRestOperationExceptio
super(
AzureServiceErrorCode.UNKNOWN.getStatusCode(),
AzureServiceErrorCode.UNKNOWN.getErrorCode(),
- "InvalidAbfsRestOperationException",
+ innerException != null
+ ? innerException.toString()
+ : "InvalidAbfsRestOperationException",
innerException);
}
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
index cd8cfa2147a..6f7ee2e1622 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
@@ -409,7 +409,9 @@ public class AbfsHttpOperation implements AbfsPerfLoggable {
}
}
} catch (IOException ex) {
- LOG.error("UnexpectedError: ", ex);
+ LOG.warn("IO/Network error: {} {}: {}",
+ method, getMaskedUrl(), ex.getMessage());
+ LOG.debug("IO Error: ", ex);
throw ex;
} finally {
if (this.isTraceEnabled) {
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index 2c669e9e101..00da9b66013 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -303,7 +303,7 @@ public class AbfsRestOperation {
} catch (UnknownHostException ex) {
String hostname = null;
hostname = httpOperation.getHost();
- LOG.warn("Unknown host name: %s. Retrying to resolve the host name...",
+ LOG.warn("Unknown host name: {}. Retrying to resolve the host name...",
hostname);
if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) {
throw new InvalidAbfsRestOperationException(ex);
@@ -311,7 +311,7 @@ public class AbfsRestOperation {
return false;
} catch (IOException ex) {
if (LOG.isDebugEnabled()) {
- LOG.debug("HttpRequestFailure: {}, {}", httpOperation.toString(), ex);
+ LOG.debug("HttpRequestFailure: {}, {}", httpOperation, ex);
}
if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) {
@@ -323,7 +323,7 @@ public class AbfsRestOperation {
intercept.updateMetrics(operationType, httpOperation);
}
- LOG.debug("HttpRequest: {}: {}", operationType, httpOperation.toString());
+ LOG.debug("HttpRequest: {}: {}", operationType, httpOperation);
if (client.getRetryPolicy().shouldRetry(retryCount, httpOperation.getStatusCode())) {
return false;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
index 0f91afe0982..6f194270e21 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferManager.java
@@ -455,8 +455,8 @@ final class ReadBufferManager {
*/
void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) {
if (LOGGER.isTraceEnabled()) {
- LOGGER.trace("ReadBufferWorker completed file {} for offset {} bytes {}",
- buffer.getStream().getPath(), buffer.getOffset(), bytesActuallyRead);
+ LOGGER.trace("ReadBufferWorker completed read file {} for offset {} outcome {} bytes {}",
+ buffer.getStream().getPath(), buffer.getOffset(), result, bytesActuallyRead);
}
synchronized (this) {
// If this buffer has already been purged during
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferWorker.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferWorker.java
index 21c9d1be766..a30f06261ef 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferWorker.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ReadBufferWorker.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.azurebfs.services;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
+import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus;
class ReadBufferWorker implements Runnable {
@@ -73,8 +74,11 @@ class ReadBufferWorker implements Runnable {
buffer.getTracingContext());
bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead); // post result back to ReadBufferManager
+ } catch (IOException ex) {
+ buffer.setErrException(ex);
+ bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0);
} catch (Exception ex) {
- buffer.setErrException(new IOException(ex));
+ buffer.setErrException(new PathIOException(buffer.getStream().getPath(), ex));
bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org