You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2022/02/06 09:14:54 UTC
[hadoop] 01/02: HDFS-16403. Improve FUSE IO performance by supporting FUSE parameter max_background (#3842)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 337e689711e5a610ec00ad4fcaede162681bef55
Author: daimin <da...@outlook.com>
AuthorDate: Tue Jan 25 13:02:37 2022 +0800
HDFS-16403. Improve FUSE IO performance by supporting FUSE parameter max_background (#3842)
Reviewed-by: Istvan Fajth <pi...@apache.org>
Reviewed-by: Wei-Chiu Chuang <we...@apache.org>
(cherry picked from commit d69938994eec9c835906d28b0ae755dfd6698dcd)
(cherry picked from commit f4241834cf4bf3cdc43a9c012a4d526b0299d88e)
---
.../src/main/native/fuse-dfs/doc/README | 1 +
.../src/main/native/fuse-dfs/fuse_dfs.c | 6 ++++++
.../src/main/native/fuse-dfs/fuse_init.c | 4 ++--
.../src/main/native/fuse-dfs/fuse_options.c | 11 +++++++----
.../src/main/native/fuse-dfs/fuse_options.h | 1 +
.../src/main/native/fuse-dfs/test/TestFuseDFS.java | 1 +
6 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
index e8cc0e5..0dc1721 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
@@ -88,6 +88,7 @@ rw
-odebug (do not daemonize - aka -d in fuse speak)
-obig_writes (use fuse big_writes option so as to allow better performance of writes on kernels >= 2.6.26)
-initchecks - have fuse-dfs try to connect to hdfs to ensure all is ok upon startup. recommended to have this on
+-omax_background=%d (maximum number of pending "background" requests - see fuse docs)
The defaults are:
entry,attribute_timeouts = 60 seconds
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c
index f693032..b9b0100 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c
@@ -81,6 +81,7 @@ int main(int argc, char *argv[])
options.rdbuffer_size = 10*1024*1024;
options.attribute_timeout = 60;
options.entry_timeout = 60;
+ options.max_background = 0;
if (-1 == fuse_opt_parse(&args, &options, dfs_opts, dfs_options)) {
return -1;
@@ -114,6 +115,11 @@ int main(int argc, char *argv[])
snprintf(buf, sizeof buf, "-oentry_timeout=%d",options.entry_timeout);
fuse_opt_add_arg(&args, buf);
+
+ if (options.max_background > 0) {
+ snprintf(buf, sizeof buf, "-omax_background=%d",options.max_background);
+ fuse_opt_add_arg(&args, buf);
+ }
}
if (options.nn_uri == NULL) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c
index 4da6da0..9799c24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c
@@ -91,11 +91,11 @@ static void dfsPrintOptions(FILE *fp, const struct options *o)
INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, "
"debug=%d, read_only=%d, initchecks=%d, "
"no_permissions=%d, usetrash=%d, entry_timeout=%d, "
- "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]",
+ "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d, max_background=%d ]",
(o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port,
o->debug, o->read_only, o->initchecks,
o->no_permissions, o->usetrash, o->entry_timeout,
- o->attribute_timeout, o->rdbuffer_size, o->direct_io);
+ o->attribute_timeout, o->rdbuffer_size, o->direct_io, o->max_background);
}
void *dfs_init(struct fuse_conn_info *conn)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c
index 8461ce4..b4082c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c
@@ -37,11 +37,13 @@ void print_options() {
"\tentry_timeout=%d\n"
"\tattribute_timeout=%d\n"
"\tprivate=%d\n"
- "\trdbuffer_size=%d (KBs)\n",
- options.protected, options.nn_uri, options.nn_port, options.debug,
+ "\trdbuffer_size=%d (KBs)\n"
+ "\tmax_background=%d\n",
+ options.protected, options.nn_uri, options.nn_port, options.debug,
options.read_only, options.usetrash, options.entry_timeout,
options.attribute_timeout, options.private,
- (int)options.rdbuffer_size / 1024);
+ (int)options.rdbuffer_size / 1024,
+ options.max_background);
}
const char *program;
@@ -56,7 +58,7 @@ void print_usage(const char *pname)
"[-ousetrash] [-obig_writes] [-oprivate (single user)] [ro] "
"[-oserver=<hadoop_servername>] [-oport=<hadoop_port>] "
"[-oentry_timeout=<secs>] [-oattribute_timeout=<secs>] "
- "[-odirect_io] [-onopoermissions] [-o<other fuse option>] "
+ "[-odirect_io] [-onopoermissions] [-omax_background=<size>] [-o<other fuse option>] "
"<mntpoint> [fuse options]\n", pname);
printf("NOTE: debugging option for fuse is -debug\n");
}
@@ -87,6 +89,7 @@ struct fuse_opt dfs_opts[] =
DFSFS_OPT_KEY("protected=%s", protected, 0),
DFSFS_OPT_KEY("port=%d", nn_port, 0),
DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0),
+ DFSFS_OPT_KEY("max_background=%d", max_background, 0),
FUSE_OPT_KEY("private", KEY_PRIVATE),
FUSE_OPT_KEY("ro", KEY_RO),
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h
index 4bfc235..2d00f1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h
@@ -34,6 +34,7 @@ struct options {
int private;
size_t rdbuffer_size;
int direct_io;
+ int max_background;
} options;
extern struct fuse_opt dfs_opts[];
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java
index dabbe00..ec5c4e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java
@@ -187,6 +187,7 @@ public class TestFuseDFS {
"-ononempty", // Don't complain about junk in mount point
"-f", // Don't background the process
"-ordbuffer=32768", // Read buffer size in kb
+ "-omax_background=100", // Set fuse max_background=100 (12 by default)
"rw"
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org