You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/09/24 18:46:03 UTC
[39/50] [abbrv] hadoop git commit: HADOOP-15778. ABFS: Fix client
side throttling for read. Contributed by Sneha Varma.
HADOOP-15778. ABFS: Fix client side throttling for read.
Contributed by Sneha Varma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0b4624c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0b4624c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0b4624c
Branch: refs/heads/HDFS-12943
Commit: d0b4624c88fc48932a7c2800185ed48bb1c5e0fe
Parents: a5692c2
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 21 11:06:24 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 21 11:06:24 2018 +0100
----------------------------------------------------------------------
.../services/AbfsClientThrottlingIntercept.java | 22 ++++++++++++++++++--
.../fs/azurebfs/services/AbfsRestOperation.java | 3 ++-
2 files changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0b4624c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java
index 97ea2a6..1c6ce17 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java
@@ -19,9 +19,12 @@
package org.apache.hadoop.fs.azurebfs.services;
import java.net.HttpURLConnection;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
+
/**
* Throttles Azure Blob File System read and write operations to achieve maximum
* throughput by minimizing errors. The errors occur when the account ingress
@@ -37,6 +40,7 @@ import org.slf4j.LoggerFactory;
public final class AbfsClientThrottlingIntercept {
private static final Logger LOG = LoggerFactory.getLogger(
AbfsClientThrottlingIntercept.class);
+ private static final String RANGE_PREFIX = "bytes=";
private static AbfsClientThrottlingIntercept singleton = null;
private AbfsClientThrottlingAnalyzer readThrottler = null;
private AbfsClientThrottlingAnalyzer writeThrottler = null;
@@ -82,7 +86,8 @@ public final class AbfsClientThrottlingIntercept {
}
break;
case ReadFile:
- contentLength = abfsHttpOperation.getBytesReceived();
+ String range = abfsHttpOperation.getConnection().getRequestProperty(HttpHeaderConfigurations.RANGE);
+ contentLength = getContentLengthIfKnown(range);
if (contentLength > 0) {
singleton.readThrottler.addBytesTransferred(contentLength,
isFailedOperation);
@@ -114,4 +119,17 @@ public final class AbfsClientThrottlingIntercept {
break;
}
}
-}
\ No newline at end of file
+
+ private static long getContentLengthIfKnown(String range) {
+ long contentLength = 0;
+ // Format is "bytes=%d-%d"
+ if (range != null && range.startsWith(RANGE_PREFIX)) {
+ String[] offsets = range.substring(RANGE_PREFIX.length()).split("-");
+ if (offsets.length == 2) {
+ contentLength = Long.parseLong(offsets[1]) - Long.parseLong(offsets[0])
+ + 1;
+ }
+ }
+ return contentLength;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0b4624c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index 9a71879..3f5717e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -156,9 +156,10 @@ public class AbfsRestOperation {
client.getAccessToken());
}
+ AbfsClientThrottlingIntercept.sendingRequest(operationType);
+
if (hasRequestBody) {
// HttpUrlConnection requires
- AbfsClientThrottlingIntercept.sendingRequest(operationType);
httpOperation.sendRequest(buffer, bufferOffset, bufferLength);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org