You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2012/08/17 01:48:23 UTC
svn commit: r1374107 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs:
CHANGES.txt src/main/native/fuse-dfs/fuse_dfs.c
src/main/native/fuse-dfs/fuse_init.c
Author: atm
Date: Thu Aug 16 23:48:23 2012
New Revision: 1374107
URL: http://svn.apache.org/viewvc?rev=1374107&view=rev
Log:
HDFS-3808. fuse_dfs: postpone libhdfs intialization until after fork. Contributed by Colin Patrick McCabe.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1374107&r1=1374106&r2=1374107&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Aug 16 23:48:23 2012
@@ -464,6 +464,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3194. DataNode block scanner is running too frequently.
(Andy Isaacson via eli)
+ HDFS-3808. fuse_dfs: postpone libhdfs intialization until after fork.
+ (Colin Patrick McCabe via atm)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c?rev=1374107&r1=1374106&r2=1374107&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c Thu Aug 16 23:48:23 2012
@@ -24,6 +24,7 @@
#include <string.h>
#include <stdlib.h>
+#include <unistd.h>
int is_protected(const char *path) {
@@ -65,15 +66,6 @@ static struct fuse_operations dfs_oper =
.truncate = dfs_truncate,
};
-static void print_env_vars(void)
-{
- const char *cp = getenv("CLASSPATH");
- const char *ld = getenv("LD_LIBRARY_PATH");
-
- fprintf(stderr, "LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld);
- fprintf(stderr, "CLASSPATH=%s",cp == NULL ? "NULL" : cp);
-}
-
int main(int argc, char *argv[])
{
int ret;
@@ -103,7 +95,7 @@ int main(int argc, char *argv[])
}
{
- char buf[1024];
+ char buf[80];
snprintf(buf, sizeof buf, "-oattr_timeout=%d",options.attribute_timeout);
fuse_opt_add_arg(&args, buf);
@@ -114,24 +106,18 @@ int main(int argc, char *argv[])
if (options.nn_uri == NULL) {
print_usage(argv[0]);
- exit(0);
- }
-
- ret = fuseConnectInit(options.nn_uri, options.nn_port);
- if (ret) {
- ERROR("FATAL: dfs_init: fuseConnInit failed with error %d!", ret);
- print_env_vars();
- exit(EXIT_FAILURE);
- }
- if (options.initchecks == 1) {
- ret = fuseConnectTest();
- if (ret) {
- ERROR("FATAL: dfs_init: fuseConnTest failed with error %d!", ret);
- print_env_vars();
- exit(EXIT_FAILURE);
- }
+ exit(EXIT_SUCCESS);
}
+ /* Note: do not call any libhdfs functions until fuse_main has been invoked.
+ *
+ * fuse_main will daemonize this process, by calling fork(). This will cause
+ * any extant threads to be destroyed, which could cause problems if
+ * libhdfs has started some Java threads.
+ *
+ * Most initialization code should go in dfs_init, which is invoked after the
+ * fork. See HDFS-3808 for details.
+ */
ret = fuse_main(args.argc, args.argv, &dfs_oper, NULL);
fuse_opt_free_args(&args);
return ret;
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c?rev=1374107&r1=1374106&r2=1374107&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c Thu Aug 16 23:48:23 2012
@@ -26,11 +26,20 @@
#include <stdlib.h>
#include <string.h>
+static void print_env_vars(void)
+{
+ const char *cp = getenv("CLASSPATH");
+ const char *ld = getenv("LD_LIBRARY_PATH");
+
+ ERROR("LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld);
+ ERROR("CLASSPATH=%s",cp == NULL ? "NULL" : cp);
+}
+
// Hacked up function to basically do:
// protectedpaths = split(options.protected,':');
-void init_protectedpaths(dfs_context *dfs) {
-
+static void init_protectedpaths(dfs_context *dfs)
+{
char *tmp = options.protected;
// handle degenerate case up front.
@@ -39,7 +48,6 @@ void init_protectedpaths(dfs_context *df
dfs->protectedpaths[0] = NULL;
return;
}
- assert(tmp);
if (options.debug) {
print_options();
@@ -80,10 +88,10 @@ void init_protectedpaths(dfs_context *df
static void dfsPrintOptions(FILE *fp, const struct options *o)
{
- fprintf(fp, "[ protected=%s, nn_uri=%s, nn_port=%d, "
+ INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, "
"debug=%d, read_only=%d, initchecks=%d, "
"no_permissions=%d, usetrash=%d, entry_timeout=%d, "
- "attribute_timeout=%d, rdbuffer_size=%Zd, direct_io=%d ]",
+ "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]",
(o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port,
o->debug, o->read_only, o->initchecks,
o->no_permissions, o->usetrash, o->entry_timeout,
@@ -92,12 +100,14 @@ static void dfsPrintOptions(FILE *fp, co
void *dfs_init(void)
{
+ int ret;
+
//
// Create a private struct of data we will pass to fuse here and which
// will then be accessible on every call.
//
- dfs_context *dfs = (dfs_context*)malloc(sizeof(dfs_context));
- if (NULL == dfs) {
+ dfs_context *dfs = calloc(1, sizeof(*dfs));
+ if (!dfs) {
ERROR("FATAL: could not malloc dfs_context");
exit(1);
}
@@ -110,17 +120,30 @@ void *dfs_init(void)
dfs->rdbuffer_size = options.rdbuffer_size;
dfs->direct_io = options.direct_io;
- fprintf(stderr, "Mounting with options ");
dfsPrintOptions(stderr, &options);
- fprintf(stderr, "\n");
init_protectedpaths(dfs);
assert(dfs->protectedpaths != NULL);
if (dfs->rdbuffer_size <= 0) {
- DEBUG("dfs->rdbuffersize <= 0 = %ld", dfs->rdbuffer_size);
+ DEBUG("dfs->rdbuffersize <= 0 = %zd", dfs->rdbuffer_size);
dfs->rdbuffer_size = 32768;
}
+
+ ret = fuseConnectInit(options.nn_uri, options.nn_port);
+ if (ret) {
+ ERROR("FATAL: dfs_init: fuseConnectInit failed with error %d!", ret);
+ print_env_vars();
+ exit(EXIT_FAILURE);
+ }
+ if (options.initchecks == 1) {
+ ret = fuseConnectTest();
+ if (ret) {
+ ERROR("FATAL: dfs_init: fuseConnectTest failed with error %d!", ret);
+ print_env_vars();
+ exit(EXIT_FAILURE);
+ }
+ }
return (void*)dfs;
}