You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/16 00:30:40 UTC
[09/10] git commit: HDFS-6912. SharedFileDescriptorFactory should not
allocate sparse files (cmccabe)
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8008f0e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8008f0e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8008f0e8
Branch: refs/heads/HDFS-6581
Commit: 8008f0e8191b1c7adbed96ed4c380208e3a37692
Parents: 2f847b3
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Sep 15 14:47:27 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Sep 15 14:47:27 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../io/nativeio/SharedFileDescriptorFactory.c | 32 ++++++++++++++++++--
2 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8008f0e8/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c2ae5ed..6f80eb2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -708,6 +708,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11056. OsSecureRandom.setConf() might leak file descriptors (yzhang
via cmccabe)
+ HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
+ (cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8008f0e8/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
index 8368402..3a8540c 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
@@ -37,6 +37,8 @@
#include <sys/types.h>
#include <unistd.h>
+#define ZERO_FULLY_BUF_SIZE 8192
+
static pthread_mutex_t g_rand_lock = PTHREAD_MUTEX_INITIALIZER;
JNIEXPORT void JNICALL
@@ -83,6 +85,24 @@ done:
}
}
+static int zero_fully(int fd, jint length)
+{
+ char buf[ZERO_FULLY_BUF_SIZE];
+ int res;
+
+ memset(buf, 0, sizeof(buf));
+ while (length > 0) {
+ res = write(fd, buf,
+ (length > ZERO_FULLY_BUF_SIZE) ? ZERO_FULLY_BUF_SIZE : length);
+ if (res < 0) {
+ if (errno == EINTR) continue;
+ return errno;
+ }
+ length -= res;
+ }
+ return 0;
+}
+
JNIEXPORT jobject JNICALL
Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0(
JNIEnv *env, jclass clazz, jstring jprefix, jstring jpath, jint length)
@@ -136,12 +156,20 @@ Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0
(*env)->Throw(env, jthr);
goto done;
}
- if (ftruncate(fd, length) < 0) {
- jthr = newIOException(env, "ftruncate(%s, %d) failed: error %d (%s)",
+ ret = zero_fully(fd, length);
+ if (ret) {
+ jthr = newIOException(env, "zero_fully(%s, %d) failed: error %d (%s)",
path, length, ret, terror(ret));
(*env)->Throw(env, jthr);
goto done;
}
+ if (lseek(fd, 0, SEEK_SET) < 0) {
+ ret = errno;
+ jthr = newIOException(env, "lseek(%s, 0, SEEK_SET) failed: error %d (%s)",
+ path, ret, terror(ret));
+ (*env)->Throw(env, jthr);
+ goto done;
+ }
jret = fd_create(env, fd); // throws exception on error.
done: