You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2016/10/31 20:55:30 UTC
hadoop git commit: HADOOP-13680. fs.s3a.readahead.range to use
getLongBytes. Contributed by Abhishek Modi.
Repository: hadoop
Updated Branches:
refs/heads/trunk f3eb4c3c7 -> a1761a841
HADOOP-13680. fs.s3a.readahead.range to use getLongBytes. Contributed by Abhishek Modi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1761a84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1761a84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1761a84
Branch: refs/heads/trunk
Commit: a1761a841e95ef7d2296ac3e40b3a26d97787eab
Parents: f3eb4c3
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 31 20:52:49 2016 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 31 20:54:46 2016 +0000
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 18 ++++++++++-----
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 17 ++++++++++++---
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 23 +++++++++++++++++++-
.../src/site/markdown/tools/hadoop-aws/index.md | 22 +++++++++++--------
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 13 ++++++++++-
5 files changed, 73 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1761a84/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index dbbb3e1..1e15b8e 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1055,8 +1055,10 @@
<property>
<name>fs.s3a.multipart.size</name>
- <value>104857600</value>
- <description>How big (in bytes) to split upload or copy operations up into.</description>
+ <value>100M</value>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
</property>
<property>
@@ -1064,7 +1066,8 @@
<value>2147483647</value>
<description>How big (in bytes) to split upload or copy operations up into.
This also controls the partition size in renamed files, as rename() involves
- copying the source file(s)
+ copying the source file(s).
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
@@ -1120,8 +1123,9 @@
<property>
<name>fs.s3a.block.size</name>
- <value>33554432</value>
+ <value>32M</value>
<description>Block size to use when reading files using s3a: file system.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
@@ -1183,10 +1187,12 @@
<property>
<name>fs.s3a.readahead.range</name>
- <value>65536</value>
+ <value>64K</value>
<description>Bytes to read ahead during a seek() before closing and
re-opening the S3 HTTP connection. This option will be overridden if
- any call to setReadahead() is made to an open stream.</description>
+ any call to setReadahead() is made to an open stream.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1761a84/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 6030fe4..925abbc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -183,10 +183,11 @@ public class S3AFileSystem extends FileSystem {
MIN_MULTIPART_THRESHOLD, DEFAULT_MIN_MULTIPART_THRESHOLD);
//check but do not store the block size
- longOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1);
+ longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1);
enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
- readAhead = longOption(conf, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE, 0);
+ readAhead = longBytesOption(conf, READAHEAD_RANGE,
+ DEFAULT_READAHEAD_RANGE, 0);
storageStatistics = (S3AStorageStatistics)
GlobalStorageStatistics.INSTANCE
.put(S3AStorageStatistics.NAME,
@@ -358,6 +359,16 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Returns the read ahead range value used by this filesystem
+ * @return
+ */
+
+ @VisibleForTesting
+ long getReadAheadRange() {
+ return readAhead;
+ }
+
+ /**
* Get the input policy for this FS instance.
* @return the input policy
*/
@@ -1883,7 +1894,7 @@ public class S3AFileSystem extends FileSystem {
*/
@Deprecated
public long getDefaultBlockSize() {
- return getConf().getLong(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE);
+ return getConf().getLongBytes(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1761a84/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 56e0c37..49f8862 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -510,6 +510,27 @@ public final class S3AUtils {
}
/**
+ * Get a long option >= the minimum allowed value, supporting memory
+ * prefixes K,M,G,T,P.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static long longBytesOption(Configuration conf,
+ String key,
+ long defVal,
+ long min) {
+ long v = conf.getLongBytes(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ return v;
+ }
+
+ /**
* Get a size property from the configuration: this property must
* be at least equal to {@link Constants#MULTIPART_MIN_SIZE}.
* If it is too small, it is rounded up to that minimum, and a warning
@@ -521,7 +542,7 @@ public final class S3AUtils {
*/
public static long getMultipartSizeProperty(Configuration conf,
String property, long defVal) {
- long partSize = conf.getLong(property, defVal);
+ long partSize = conf.getLongBytes(property, defVal);
if (partSize < MULTIPART_MIN_SIZE) {
LOG.warn("{} must be at least 5 MB; configured value is {}",
property, partSize);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1761a84/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 0eb36ef..54a4ba9 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -762,16 +762,20 @@ from placing its declaration on the command line.
<property>
<name>fs.s3a.multipart.size</name>
- <value>104857600</value>
+ <value>100M</value>
<description>How big (in bytes) to split upload or copy operations up into.
- This also controls the partition size in renamed files, as rename() involves
- copying the source file(s)</description>
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
</property>
<property>
<name>fs.s3a.multipart.threshold</name>
<value>2147483647</value>
- <description>Threshold before uploads or copies use parallel multipart operations.</description>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ This also controls the partition size in renamed files, as rename() involves
+ copying the source file(s).
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
+ </description>
</property>
<property>
@@ -825,7 +829,7 @@ from placing its declaration on the command line.
<property>
<name>fs.s3a.block.size</name>
- <value>33554432</value>
+ <value>32M</value>
<description>Block size to use when reading files using s3a: file system.
</description>
</property>
@@ -859,7 +863,7 @@ from placing its declaration on the command line.
<property>
<name>fs.s3a.readahead.range</name>
- <value>65536</value>
+ <value>64K</value>
<description>Bytes to read ahead during a seek() before closing and
re-opening the S3 HTTP connection. This option will be overridden if
any call to setReadahead() is made to an open stream.</description>
@@ -1029,9 +1033,9 @@ S3 endpoints, as disks are not used for intermediate data storage.
<property>
<name>fs.s3a.multipart.size</name>
- <value>104857600</value>
- <description>
- How big (in bytes) to split upload or copy operations up into.
+ <value>100M</value>
+ <description>How big (in bytes) to split upload or copy operations up into.
+ A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1761a84/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 6ae9613..9163b15 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -380,7 +380,7 @@ public class ITestS3AConfiguration {
byte[] file = ContractTestUtils.toAsciiByteArray("test file");
ContractTestUtils.writeAndRead(fs,
new Path("/path/style/access/testFile"), file, file.length,
- conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
+ (int) conf.getLongBytes(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
} catch (final AWSS3IOException e) {
LOG.error("Caught exception: ", e);
// Catch/pass standard path style access behaviour when live bucket
@@ -452,6 +452,17 @@ public class ITestS3AConfiguration {
}
@Test
+ public void testReadAheadRange() throws Exception {
+ conf = new Configuration();
+ conf.set(Constants.READAHEAD_RANGE, "300K");
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ assertNotNull(fs);
+ long readAheadRange = fs.getReadAheadRange();
+ assertNotNull(readAheadRange);
+ assertEquals("Read Ahead Range Incorrect.", 300 * 1024, readAheadRange);
+ }
+
+ @Test
public void testUsernameFromUGI() throws Throwable {
final String alice = "alice";
UserGroupInformation fakeUser =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org