You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2008/05/29 22:18:12 UTC
svn commit: r661462 - in /hadoop/core/trunk: ./ src/contrib/fuse-dfs/
src/contrib/fuse-dfs/src/ src/contrib/fuse-dfs/test/
Author: cutting
Date: Thu May 29 13:18:11 2008
New Revision: 661462
URL: http://svn.apache.org/viewvc?rev=661462&view=rev
Log:
HADOOP-4. Add fuse-dfs to contrib, permitting one to mount an HDFS filesystem on systems that support FUSE, e.g., Linux. Contributed by Pete Wyckoff.
Added:
hadoop/core/trunk/src/contrib/fuse-dfs/
hadoop/core/trunk/src/contrib/fuse-dfs/Makefile.am
hadoop/core/trunk/src/contrib/fuse-dfs/README
hadoop/core/trunk/src/contrib/fuse-dfs/acinclude.m4
hadoop/core/trunk/src/contrib/fuse-dfs/bootstrap.sh (with props)
hadoop/core/trunk/src/contrib/fuse-dfs/build.xml
hadoop/core/trunk/src/contrib/fuse-dfs/configure.ac
hadoop/core/trunk/src/contrib/fuse-dfs/global_footer.mk
hadoop/core/trunk/src/contrib/fuse-dfs/global_header.mk
hadoop/core/trunk/src/contrib/fuse-dfs/src/
hadoop/core/trunk/src/contrib/fuse-dfs/src/Makefile.am
hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs.c
hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh (with props)
hadoop/core/trunk/src/contrib/fuse-dfs/test/
hadoop/core/trunk/src/contrib/fuse-dfs/test/TestFuseDFS.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/build.xml
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=661462&r1=661461&r2=661462&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu May 29 13:18:11 2008
@@ -104,6 +104,10 @@
HADOOP-3400. A new API FileSystem.deleteOnExit() that facilitates
handling of temporary files in HDFS. (dhruba)
+ HADOOP-4. Add fuse-dfs to contrib, permitting one to mount an
+ HDFS filesystem on systems that support FUSE, e.g., Linux.
+ (Pete Wyckoff via cutting)
+
IMPROVEMENTS
HADOOP-2928. Remove deprecated FileSystem.getContentLength().
Modified: hadoop/core/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/build.xml?rev=661462&r1=661461&r2=661462&view=diff
==============================================================================
--- hadoop/core/trunk/build.xml (original)
+++ hadoop/core/trunk/build.xml Thu May 29 13:18:11 2008
@@ -367,7 +367,7 @@
depends="clover,compile-core-classes,compile-core-native,compile-c++" description="Compile core only">
</target>
- <target name="compile-contrib" depends="compile-core">
+ <target name="compile-contrib" depends="compile-core,compile-libhdfs">
<subant target="compile">
<property name="version" value="${version}"/>
<fileset file="${contrib.dir}/build.xml"/>
@@ -759,7 +759,8 @@
<!-- ================================================================== -->
<!-- -->
<!-- ================================================================== -->
- <target name="package" depends="jar, javadoc, examples, jar-test, ant-tasks" description="Build distribution">
+ <target name="package" depends="compile, jar, javadoc, examples, jar-test, ant-tasks, package-libhdfs"
+ description="Build distribution">
<mkdir dir="${dist.dir}"/>
<mkdir dir="${dist.dir}/lib"/>
<mkdir dir="${dist.dir}/contrib"/>
@@ -910,7 +911,7 @@
<!-- ================================================================== -->
<!-- libhdfs targets. -->
<!-- ================================================================== -->
- <target name="compile-libhdfs" depends="init">
+ <target name="compile-libhdfs" depends="init" if="libhdfs">
<mkdir dir="${build.libhdfs}"/>
<exec dir="${libhdfs.src}" executable="${make.cmd}" failonerror="true">
<env key="OS_NAME" value="${os.name}"/>
@@ -944,7 +945,9 @@
</exec>
</target>
- <target name="package-libhdfs" depends="compile-libhdfs, doc-libhdfs, package">
+ <target name="package-libhdfs"
+ depends="compile-libhdfs, doc-libhdfs"
+ if="libhdfs">
<mkdir dir="${dist.dir}/libhdfs"/>
<copy todir="${dist.dir}/libhdfs">
<fileset dir="${build.libhdfs}"
Added: hadoop/core/trunk/src/contrib/fuse-dfs/Makefile.am
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/Makefile.am?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/Makefile.am (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/Makefile.am Thu May 29 13:18:11 2008
@@ -0,0 +1,27 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+@GLOBAL_HEADER_MK@
+
+@PRODUCT_MK@
+
+SUBDIRS = . src
+
+clean:
+ rm -rf autom4te.cache config.guess config.log config.status config.sub configure depcomp src/.deps install-sh Makefile.in src/Makefile.in src/Makefile missing Makefile src/fuse_dfs.o src/fuse_dfs aclocal.m4
+
+
+@GLOBAL_FOOTER_MK@
Added: hadoop/core/trunk/src/contrib/fuse-dfs/README
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/README?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/README (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/README Thu May 29 13:18:11 2008
@@ -0,0 +1,98 @@
+# Copyright 2005 The Apache Software Foundation Licensed under the
+# Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License. You may obtain a copy
+# of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
+# required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+This is a FUSE module for Hadoop's HDFS.
+
+It allows one to mount HDFS as a Unix filesystem and optionally export
+that mount point to other machines.
+
+For now, writes are disabled as this requires Hadoop-1700 - file
+appends which I guess won't be ready till 0.18 ish ??.
+
+rmdir, mv, mkdir, rm are all supported. just not cp, touch, ...
+
+BUILDING:
+
+
+Requirements:
+
+ 1. a Linux kernel > 2.6.9 or a kernel module from FUSE - i.e., you
+ compile it yourself and then modprobe it. Better off with the
+ former option if possible. (Note for now if you use the kernel
+ with fuse included, it doesn't allow you to export this through NFS
+ so be warned. See the FUSE email list for more about this.)
+
+ 2. FUSE should be installed in /usr/local or FUSE_HOME ant
+ environment variable
+
+To build:
+
+ 1. in HADOOP_HOME: ant compile-contrib -Dcompile.c++=1 -Dfusedfs=1
+
+
+NOTE: for amd64 architecture, libhdfs will not compile unless you edit
+the Makefile in src/c++/libhdfs/Makefile and set OS_ARCH=amd64
+(probably the same for others too).
+
+--------------------------------------------------------------------------------
+
+CONFIGURING:
+
+Look at all the paths in fuse_dfs_wrapper.sh and either correct them
+or set them in your environment before running. (note for automount
+and mount as root, you probably cannnot control the environment, so
+best to set them in the wrapper)
+
+INSTALLING:
+
+1. mkdir /mnt/dfs (or wherever you want to mount it)
+
+2. fuse_dfs_wrapper.sh dfs://hadoop_server1.foo.com:9000 /mnt/dfs -d
+; and from another terminal, try ls /mnt/dfs
+
+If 2 works, try again dropping the debug mode, i.e., -d
+
+(note - common problems are that you don't have libhdfs.so or
+libjvm.so or libfuse.so on your LD_LIBRARY_PATH, and your CLASSPATH
+does not contain hadoop and other required jars.)
+
+--------------------------------------------------------------------------------
+
+
+DEPLOYING:
+
+in a root shell do the following:
+
+1. add the following to /etc/fstab -
+ fuse_dfs#dfs://hadoop_server.foo.com:9000 /mnt/dfs fuse
+ allow_other,rw 0 0
+
+2. mount /mnt/dfs Expect problems with not finding fuse_dfs. You will
+ need to probably add this to /sbin and then problems finding the
+ above 3 libraries. Add these using ldconfig.
+
+--------------------------------------------------------------------------------
+
+
+EXPORTING:
+
+Add the following to /etc/exports:
+
+ /mnt/hdfs *.foo.com(no_root_squash,rw,fsid=1,sync)
+
+NOTE - you cannot export this with a FUSE module built into the kernel
+- e.g., kernel 2.6.17. For info on this, refer to the FUSE wiki.
+--------------------------------------------------------------------------------
+
+ADVANCED:
+
+you may want to ensure certain directories cannot be deleted from the
+shell until the FS has permissions. You can set this in the build.xml
+file in src/contrib/fuse-dfs/build.xml
Added: hadoop/core/trunk/src/contrib/fuse-dfs/acinclude.m4
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/acinclude.m4?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/acinclude.m4 (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/acinclude.m4 Thu May 29 13:18:11 2008
@@ -0,0 +1,268 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+AC_DEFUN([FB_INITIALIZE],
+[
+AM_INIT_AUTOMAKE([ foreign 1.9.5 no-define ])
+if test "x$1" = "xlocalinstall"; then
+wdir=`pwd`
+# To use $wdir undef quote.
+#
+##########
+AC_PREFIX_DEFAULT([`pwd`/install])
+echo
+fi
+AC_PROG_CC
+AC_PROG_CXX
+AC_PROG_RANLIB(RANLIB, ranlib)
+AC_PATH_PROGS(BASH, bash)
+AC_PATH_PROGS(PERL, perl)
+AC_PATH_PROGS(PYTHON, python)
+AC_PATH_PROGS(AR, ar)
+AC_PATH_PROGS(ANT, ant)
+PRODUCT_MK=""
+])
+
+AC_DEFUN([FB_WITH_EXTERNAL_PATH],
+[
+cdir=`pwd`
+AC_MSG_CHECKING([Checking EXTERNAL_PATH set to])
+AC_ARG_WITH([externalpath],
+ [ --with-externalpath=DIR User specified path to external facebook components.],
+ [
+ if test "x${EXTERNAL_PATH}" != "x"; then
+ echo ""
+ echo "ERROR: You have already set EXTERNAL_PATH in your environment"
+ echo "Cannot override it using --with-externalpath. Unset EXTERNAL_PATH to use this option"
+ exit 1
+ fi
+ EXTERNAL_PATH=$withval
+ ],
+ [
+ if test "x${EXTERNAL_PATH}" = "x"; then
+ EXTERNAL_PATH=$1
+ fi
+ ]
+)
+if test "x${EXTERNAL_PATH}" = "x"; then
+ export EXTERNAL_PATH="$cdir/external"
+ GLOBAL_HEADER_MK="include ${EXTERNAL_PATH}/global_header.mk"
+ GLOBAL_FOOTER_MK="include ${EXTERNAL_PATH}/global_footer.mk"
+else
+ export EXTERNAL_PATH
+ GLOBAL_HEADER_MK="include ${EXTERNAL_PATH}/global_header.mk"
+ GLOBAL_FOOTER_MK="include ${EXTERNAL_PATH}/global_footer.mk"
+fi
+AC_MSG_RESULT($EXTERNAL_PATH)
+if test ! -d ${EXTERNAL_PATH}; then
+ echo ""
+ echo "ERROR: EXTERNAL_PATH set to an nonexistent directory ${EXTERNAL_PATH}"
+ exit 1
+fi
+AC_SUBST(EXTERNAL_PATH)
+AC_SUBST(GLOBAL_HEADER_MK)
+AC_SUBST(GLOBAL_FOOTER_MK)
+])
+
+# Set option to enable shared mode. Set DEBUG and OPT for use in Makefile.am.
+AC_DEFUN([FB_ENABLE_DEFAULT_OPT_BUILD],
+[
+AC_MSG_CHECKING([whether to enable optimized build])
+AC_ARG_ENABLE([opt],
+ [ --disable-opt Set up debug mode.],
+ [
+ ENABLED_OPT=$enableval
+ ],
+ [
+ ENABLED_OPT="yes"
+ ]
+)
+if test "$ENABLED_OPT" = "yes"
+then
+ CFLAGS="-Wall -O3"
+ CXXFLAGS="-Wall -O3"
+else
+ CFLAGS="-Wall -g"
+ CXXFLAGS="-Wall -g"
+fi
+AC_MSG_RESULT($ENABLED_OPT)
+AM_CONDITIONAL([OPT], [test "$ENABLED_OPT" = yes])
+AM_CONDITIONAL([DEBUG], [test "$ENABLED_OPT" = no])
+])
+
+# Set option to enable debug mode. Set DEBUG and OPT for use in Makefile.am.
+AC_DEFUN([FB_ENABLE_DEFAULT_DEBUG_BUILD],
+[
+AC_MSG_CHECKING([whether to enable debug build])
+AC_ARG_ENABLE([debug],
+ [ --disable-debug Set up opt mode.],
+ [
+ ENABLED_DEBUG=$enableval
+ ],
+ [
+ ENABLED_DEBUG="yes"
+ ]
+)
+if test "$ENABLED_DEBUG" = "yes"
+then
+ CFLAGS="-Wall -g"
+ CXXFLAGS="-Wall -g"
+else
+ CFLAGS="-Wall -O3"
+ CXXFLAGS="-Wall -O3"
+fi
+AC_MSG_RESULT($ENABLED_DEBUG)
+AM_CONDITIONAL([DEBUG], [test "$ENABLED_DEBUG" = yes])
+AM_CONDITIONAL([OPT], [test "$ENABLED_DEBUG" = no])
+])
+
+# Set option to enable static libs.
+AC_DEFUN([FB_ENABLE_DEFAULT_STATIC],
+[
+SHARED=""
+STATIC=""
+AC_MSG_CHECKING([whether to enable static mode])
+AC_ARG_ENABLE([static],
+ [ --disable-static Set up shared mode.],
+ [
+ ENABLED_STATIC=$enableval
+ ],
+ [
+ ENABLED_STATIC="yes"
+ ]
+)
+if test "$ENABLED_STATIC" = "yes"
+then
+ LTYPE=".a"
+else
+ LTYPE=".so"
+ SHARED_CXXFLAGS="-fPIC"
+ SHARED_CFLAGS="-fPIC"
+ SHARED_LDFLAGS="-shared -fPIC"
+ AC_SUBST(SHARED_CXXFLAGS)
+ AC_SUBST(SHARED_CFLAGS)
+ AC_SUBST(SHARED_LDFLAGS)
+fi
+AC_MSG_RESULT($ENABLED_STATIC)
+AC_SUBST(LTYPE)
+AM_CONDITIONAL([STATIC], [test "$ENABLED_STATIC" = yes])
+AM_CONDITIONAL([SHARED], [test "$ENABLED_STATIC" = no])
+])
+
+# Set option to enable shared libs.
+AC_DEFUN([FB_ENABLE_DEFAULT_SHARED],
+[
+SHARED=""
+STATIC=""
+AC_MSG_CHECKING([whether to enable shared mode])
+AC_ARG_ENABLE([shared],
+ [ --disable-shared Set up static mode.],
+ [
+ ENABLED_SHARED=$enableval
+ ],
+ [
+ ENABLED_SHARED="yes"
+ ]
+)
+if test "$ENABLED_SHARED" = "yes"
+then
+ LTYPE=".so"
+ SHARED_CXXFLAGS="-fPIC"
+ SHARED_CFLAGS="-fPIC"
+ SHARED_LDFLAGS="-shared -fPIC"
+ AC_SUBST(SHARED_CXXFLAGS)
+ AC_SUBST(SHARED_CFLAGS)
+ AC_SUBST(SHARED_LDFLAGS)
+else
+ LTYPE=".a"
+fi
+AC_MSG_RESULT($ENABLED_SHARED)
+AC_SUBST(LTYPE)
+AM_CONDITIONAL([SHARED], [test "$ENABLED_SHARED" = yes])
+AM_CONDITIONAL([STATIC], [test "$ENABLED_SHARED" = no])
+])
+
+# Generates define flags and conditionals as specified by user.
+# This gets enabled *only* if user selects --enable-<FEATURE> otion.
+AC_DEFUN([FB_ENABLE_FEATURE],
+[
+ENABLE=""
+flag="$1"
+value="$3"
+AC_MSG_CHECKING([whether to enable $1])
+AC_ARG_ENABLE([$2],
+ [ --enable-$2 Enable $2.],
+ [
+ ENABLE=$enableval
+ ],
+ [
+ ENABLE="no"
+ ]
+)
+AM_CONDITIONAL([$1], [test "$ENABLE" = yes])
+if test "$ENABLE" = "yes"
+then
+ if test "x${value}" = "x"
+ then
+ AC_DEFINE([$1])
+ else
+ AC_DEFINE_UNQUOTED([$1], [$value])
+ fi
+fi
+AC_MSG_RESULT($ENABLE)
+])
+
+
+# can also use eval $2=$withval;AC_SUBST($2)
+AC_DEFUN([FB_WITH_PATH],
+[
+USRFLAG=""
+USRFLAG=$1
+AC_MSG_CHECKING([Checking $1 set to])
+AC_ARG_WITH([$2],
+ [ --with-$2=DIR User specified path.],
+ [
+ LOC=$withval
+ eval $USRFLAG=$withval
+ ],
+ [
+ LOC=$3
+ eval $USRFLAG=$3
+ ]
+)
+AC_SUBST([$1])
+AC_MSG_RESULT($LOC)
+])
+
+AC_DEFUN([FB_SET_FLAG_VALUE],
+[
+SETFLAG=""
+AC_MSG_CHECKING([Checking $1 set to])
+SETFLAG=$1
+eval $SETFLAG=\"$2\"
+AC_SUBST([$SETFLAG])
+AC_MSG_RESULT($2)
+])
+
+# NOTES
+# if using if else bourne stmt you must have more than a macro in it.
+# EX1 is not correct. EX2 is correct
+# EX1: if test "$XX" = "yes"; then
+# AC_SUBST(xx)
+# fi
+# EX2: if test "$XX" = "yes"; then
+# xx="foo"
+# AC_SUBST(xx)
+# fi
Added: hadoop/core/trunk/src/contrib/fuse-dfs/bootstrap.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/bootstrap.sh?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/bootstrap.sh (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/bootstrap.sh Thu May 29 13:18:11 2008
@@ -0,0 +1,21 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#!/bin/sh
+
+aclocal
+automake -a
+autoconf
+./configure
Propchange: hadoop/core/trunk/src/contrib/fuse-dfs/bootstrap.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/fuse-dfs/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/build.xml?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/build.xml (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/build.xml Thu May 29 13:18:11 2008
@@ -0,0 +1,77 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="fuse-dfs" default="compile">
+
+ <import file="../build-contrib.xml"/>
+
+ <target name="check-libhdfs-fuse">
+ <condition property="libhdfs-fuse">
+ <and>
+ <isset property="fusedfs"/>
+ <isset property="libhdfs"/>
+ </and>
+ </condition>
+ </target>
+
+ <!-- override compile target !-->
+ <target name="compile" depends="check-libhdfs-fuse" if="libhdfs-fuse">
+ <echo message="contrib: ${name}"/>
+
+ <exec executable="/bin/sh" failonerror="true">
+ <arg value="${root}/bootstrap.sh"/>
+ </exec>
+
+ <exec executable="make" failonerror="true">
+ <env key="OS_NAME" value="${os.name}"/>
+ <env key="OS_ARCH" value="${os.arch}"/>
+ <env key="HADOOP_HOME" value="${hadoop.root}"/>
+ <env key="PROTECTED_PATHS" value="/,/Trash,/user"/>
+ <env key="PACKAGE_VERSION" value="0.1.0"/>
+ <env key="FUSE_HOME" value="/usr/local"/>
+ </exec>
+ </target>
+
+ <!-- override jar target !-->
+ <target name="jar"/>
+
+ <!-- override package target !-->
+ <target name="package" depends="check-libhdfs-fuse" if="libhdfs-fuse">
+ <echo message="contrib: ${name}"/>
+
+ <mkdir dir="${dist.dir}/contrib/${name}"/>
+ <exec executable="cp">
+ <arg value="-p"/>
+ <arg value="README"/>
+ <arg value="src/fuse_dfs"/>
+ <arg value="src/fuse_dfs_wrapper.sh"/>
+ <arg value="${dist.dir}/contrib/${name}"/>
+ </exec>
+ </target>
+
+ <!-- override clean target !-->
+ <target name="clean" depends="check-libhdfs-fuse" if="libhdfs-fuse">
+ <echo message="contrib: ${name}"/>
+
+ <exec executable="make">
+ <arg value="clean"/>
+ </exec>
+ </target>
+
+</project>
Added: hadoop/core/trunk/src/contrib/fuse-dfs/configure.ac
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/configure.ac?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/configure.ac (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/configure.ac Thu May 29 13:18:11 2008
@@ -0,0 +1,135 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Autoconf input file
+# $Id$
+
+# AC - autoconf
+# FB - facebook
+
+#########################################################################
+# Section 1:
+# DO NOT TOUCH EXCEPT TO CHANGE Product-Name and Rev# IN AC_INIT
+
+AC_PREREQ(2.52)
+AC_INIT([fuse_dfs], [0.1.0])
+#AC_CONFIG_AUX_DIR([/usr/share/automake-1.9])
+# To install locally
+AC_CANONICAL_TARGET()
+FB_INITIALIZE([localinstall])
+AC_PREFIX_DEFAULT([`pwd`])
+
+
+#case $target in
+#*64*intel)
+# OS_ARCH=intel64 ;;
+#*64*amd* | *64*unknown*)
+# OS_ARCH=amd64 ;;
+#$esac
+#AC_SUBST(OS_ARCH)
+ DEFS=""
+AC_SUBST([DEFS])
+
+############################################################################
+# Section 2:
+# User Configurable system defaults. Change With CAUTION!
+
+# User can include custom makefile rules. Uncomment and update only <name> in PRODUCT_MK.
+# Include where appropriate in any Makefile.am as @PRODUCT_MK@
+
+# Default path to external Facebook components and shared build tools I.e fb303 etc.
+# To point to other locations set environment variable EXTERNAL_PATH.
+# DO NOT change default. Changing default value requires changing bootstrap.sh.
+FB_WITH_EXTERNAL_PATH([`pwd`])
+
+# Pre-defined macro to set optimized build mode. Configure with --disable-opt option to turn off optimization. Default CXXFLAGS set to '-Wall -O3'. In debug mode CXXFLAGS is '-Wall -g'
+# FB_ENABLE_DEFAULT_DEBUG_BUILD
+FB_ENABLE_DEFAULT_OPT_BUILD
+
+# Predefined macro to set static library mode. Configure with --disable-static option to turn off static lib mode.
+# FB_ENABLE_DEFAULT_SHARED
+FB_ENABLE_DEFAULT_STATIC
+
+##########################################################################
+# Section 3:
+# User Configurable
+
+# Personalized FLAG setting macro. Sets FLAG to user specifed value overriding any default.
+# $(FLAG) can be used in Makefile.am and global mk's.
+# FB_SET_FLAG_VALUE([<FLAGNAME>], [<value>])
+# FB_SET_FLAG_VALUE([CXXFLAGS], [-g -O3])
+
+# Personalized feature generator. Creates defines/conditionals and --enable --disable command line options. Doesn't enable until configured with --enable-<feature> option.
+# FB_ENABLE_FEATURE([FEATURE], [feature]) OR FB_ENABLE_FEATURE([FEATURE], [feature], [\"<value>\"])
+# Example: Macro supplies -DFACEBOOK at compile time and "if FACEBOOK endif" capabilities.
+
+# FB_ENABLE_FEATURE([FACEBOOK], [facebook])
+#FB_ENABLE_FEATURE([HDFS], [hdfs])
+
+# Personalized path generator Sets default paths. Provides --with-xx=DIR options.
+# FB_WITH_PATH([<var>_home], [<var>path], [<default location>]
+# Example: sets $(thrift_dir) variable with default path.
+#FB_WITH_PATH([HADOOP_HOME], [hadooppath], [/usr/local/share/hadoop/])
+#FB_WITH_PATH([external_home], [externalpath], [${EXTERNAL_PATH}/external])
+#FB_WITH_PATH([FUSE_HOME], [fusehome], [/usr/local])
+#FB_WITH_PATH([JAVA_HOME], [jdkhome], [/usr/local/jdk])
+#FB_WITH_PATH([PROTECTED_PATHS], [protectedpaths], [/,/Trash,/user])
+
+# Set default location of "php-config". User can also use "--with-php-config=<path>" to point to another location.
+# Run "./configure --help" to see --with options.
+# PHPCONFIGLIBS and PHPCONFIGINCS are generated.
+# FB_SET_PHP([PHPCONFIGDIR], [/usr/local/bin])
+
+# Generates Makefile from Makefile.am. Modify when new subdirs are added.
+# Change Makefile.am also to add subdirectly.
+AC_CONFIG_FILES(Makefile src/Makefile)
+
+
+############################################################################
+# Section 4:
+# DO NOT TOUCH.
+
+AC_SUBST(PRODUCT_MK)
+AC_OUTPUT
+
+#############################################################################
+######### FINISH ############################################################
+
+echo "EXTERNAL_PATH $EXTERNAL_PATH"
+
+
+#
+# NOTES FOR USER
+# Short cut to create conditional flags.
+#enable_facebook="yes"
+#AM_CONDITIONAL([FACEBOOK], [test "$enable_facebook" = yes])
+#enable_hdfs="yes"
+#AM_CONDITIONAL([HDFS], [test "$enable_hdfs" = yes])
+
+# Enable options with --enable and --disable configurable.
+#AC_MSG_CHECKING([whether to enable FACEBOOK])
+#FACEBOOK=""
+#AC_ARG_ENABLE([facebook],
+# [ --enable-facebook Enable facebook.],
+# [
+# ENABLE_FACEBOOK=$enableval
+# ],
+# [
+# ENABLE_FACEBOOK="no"
+# ]
+#)
+#AM_CONDITIONAL([FACEBOOK], [test "$ENABLE_FACEBOOK" = yes])
+#AC_MSG_RESULT($ENABLE_FACEBOOK)
+
Added: hadoop/core/trunk/src/contrib/fuse-dfs/global_footer.mk
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/global_footer.mk?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/global_footer.mk (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/global_footer.mk Thu May 29 13:18:11 2008
@@ -0,0 +1,17 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+thriftstyle : $(XBUILT_SOURCES)
+
Added: hadoop/core/trunk/src/contrib/fuse-dfs/global_header.mk
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/global_header.mk?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/global_header.mk (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/global_header.mk Thu May 29 13:18:11 2008
@@ -0,0 +1,50 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ifneq ($$(XBUILT_SOURCES),)
+ XBUILT_SOURCES := $$(XBUILT_SOURCES) $$(XTARGET)
+else
+ XBUILT_SOURCES := $$(XTARGET)
+endif
+
+showvars:
+ @echo BUILD_SOURCES = $(BUILT_SOURCES)
+ @echo XBUILTSOURCES = $(XBUILT_SOURCES)
+ @echo DEFS = $(DEFS)
+ @echo CXXFLAGS = $(CXXFLAGS)
+ @echo AM_CXXFLAGS = $(AM_CXXFLAGS)
+ @echo CPPFLAGS = $(CPPFLAGS)
+ @echo AM_CPPFLAGS = $(AM_CPPFLAGS)
+ @echo LDFLAGS = $(LDFLAGS)
+ @echo AM_LDFLAGS = $(AM_LDFLAGS)
+ @echo LDADD = $(LDADD)
+ @echo LIBS = $(LIBS)
+ @echo EXTERNAL_LIBS = $(EXTERNAL_LIBS)
+ @echo EXTERNAL_PATH = $(EXTERNAL_PATH)
+ @echo MAKE = $(MAKE)
+ @echo MAKE_FLAGS = $(MAKE_FLAGS)
+ @echo AM_MAKEFLAGS = $(AM_MAKEFLAGS)
+ @echo top_builddir = $(top_builddir)
+ @echo top_srcdir = $(top_srcdir)
+ @echo srcdir = $(srcdir)
+ @echo PHPVAL = $(PHPVAL)
+ @echo PHPCONFIGDIR = $(PHPCONFIGDIR)
+ @echo PHPCONFIGINCLUDEDIR = $(PHPCONFIGINCLUDEDIR)
+ @echo PHPCONFIGINCLUDES = $(PHPCONFIGINCLUDES)
+ @echo PHPCONFIGLDFLAGS = $(PHPCONFIGLDFLAGS)
+ @echo PHPCONFIGLIBS = $(PHPCONFIGLIBS)
+
+clean-common:
+ rm -rf gen-*
Added: hadoop/core/trunk/src/contrib/fuse-dfs/src/Makefile.am
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/src/Makefile.am?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/src/Makefile.am (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/src/Makefile.am Thu May 29 13:18:11 2008
@@ -0,0 +1,20 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+bin_PROGRAMS = fuse_dfs
+fuse_dfs_SOURCES = fuse_dfs.c
+AM_CPPFLAGS= -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_HOME)/src/c++/libhdfs/ -I$(JAVA_HOME)/include/linux/ -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\"
+AM_LDFLAGS= -L$(HADOOP_HOME)/build/libhdfs -lhdfs -L$(FUSE_HOME)/lib -lfuse -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -ljvm
+
Added: hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs.c
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs.c?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs.c (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs.c Thu May 29 13:18:11 2008
@@ -0,0 +1,1064 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define FUSE_USE_VERSION 26
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef linux
+/* For pread()/pwrite() */
+#define _XOPEN_SOURCE 500
+#endif
+
+#include <fuse.h>
+#include <fuse/fuse_opt.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <errno.h>
+#include <sys/time.h>
+#ifdef HAVE_SETXATTR
+#include <sys/xattr.h>
+#endif
+#include <math.h> // for ceil
+#include <getopt.h>
+#include <assert.h>
+#include <syslog.h>
+#include <strings.h>
+
+#include <hdfs.h>
+
+// Constants
+//
+static const int default_id = 99; // nobody - not configurable since soon uids in dfs, yeah!
+static const size_t rd_buf_size = 128 * 1024;
+static const int blksize = 512;
+static const size_t rd_cache_buf_size = 10*1024*1024;//how much of reads to buffer here
+
+/** options for fuse_opt.h */
+struct options {
+ char* server;
+ int port;
+ int debug;
+ int nowrites;
+ int no_trash;
+}options;
+
+
+typedef struct dfs_fh_struct {
+ hdfsFile hdfsFH;
+ char *buf;
+ tSize sizeBuffer; //what is the size of the buffer we have
+ off_t startOffset; //where the buffer starts in the file
+} dfs_fh;
+
+#include <stddef.h>
+
+/** macro to define options */
+#define DFSFS_OPT_KEY(t, p, v) { t, offsetof(struct options, p), v }
+
+/** keys for FUSE_OPT_ options */
+static void print_usage(const char *pname)
+{
+ fprintf(stdout,"USAGE: %s [--debug] [--help] [--version] [--nowrites] [--notrash] --server=<hadoop_servername> --port=<hadoop_port> <mntpoint> [fuse options]\n",pname);
+ fprintf(stdout,"NOTE: a useful fuse option is -o allow_others and -o default_permissions\n");
+ fprintf(stdout,"NOTE: optimizations include -o entry_timeout=500 -o attr_timeout=500\n");
+ fprintf(stdout,"NOTE: debugging option for fuse is -debug\n");
+}
+
+
+#define OPTIMIZED_READS 1
+
+
+enum
+ {
+ KEY_VERSION,
+ KEY_HELP,
+ };
+
+static struct fuse_opt dfs_opts[] =
+ {
+ DFSFS_OPT_KEY("--server=%s", server, 0),
+ DFSFS_OPT_KEY("--port=%d", port, 0),
+ DFSFS_OPT_KEY("--debug", debug, 1),
+ DFSFS_OPT_KEY("--nowrites", nowrites, 1),
+ DFSFS_OPT_KEY("--notrash", no_trash, 1),
+
+ FUSE_OPT_KEY("-v", KEY_VERSION),
+ FUSE_OPT_KEY("--version", KEY_VERSION),
+ FUSE_OPT_KEY("-h", KEY_HELP),
+ FUSE_OPT_KEY("--help", KEY_HELP),
+ FUSE_OPT_END
+ };
+
+static const char *program;
+
+int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs)
+{
+
+ if (key == KEY_VERSION) {
+ fprintf(stdout,"%s %s\n",program,_FUSE_DFS_VERSION);
+ exit(0);
+ } else if (key == KEY_HELP) {
+ print_usage(program);
+ exit(0);
+ } else {
+ // try and see if the arg is a URI for DFS
+ int tmp_port;
+ char tmp_server[1024];
+
+ if (!sscanf(arg,"dfs://%1024[a-zA-Z0-9_.-]:%d",tmp_server,&tmp_port)) {
+ printf("didn't recognize %s\n",arg);
+ fuse_opt_add_arg(outargs,arg);
+ } else {
+ options.port = tmp_port;
+ options.server = strdup(tmp_server);
+ }
+ }
+ return 0;
+}
+
+
+//
+// Structure to store fuse_dfs specific data
+// this will be created and passed to fuse at startup
+// and fuse will pass it back to us via the context function
+// on every operation.
+//
+typedef struct dfs_context_struct {
+ int debug;
+ char *nn_hostname;
+ int nn_port;
+ hdfsFS fs;
+ int nowrites;
+ int no_trash;
+
+ // todo:
+ // total hack city - use this to strip off the dfs url from the filenames
+ // that the dfs API is now providing in 0.14.5
+ // Will do a better job of fixing this once I am back from vacation
+ //
+ char dfs_uri[1024];
+ int dfs_uri_len;
+} dfs_context;
+
+
+//
+// Start of read-only functions
+//
+
+static int dfs_getattr(const char *path, struct stat *st)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(dfs);
+ assert(path);
+ assert(st);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ // call the dfs API to get the actual information
+ hdfsFileInfo *info = hdfsGetPathInfo(dfs->fs,path);
+
+ if (NULL == info) {
+ return -ENOENT;
+ }
+
+ // initialize the stat structure
+ memset(st, 0, sizeof(struct stat));
+
+ // setup hard link info - for a file it is 1 else num entries in a dir + 2 (for . and ..)
+ if (info[0].mKind == kObjectKindDirectory) {
+ int numEntries = 0;
+ hdfsFileInfo *info = hdfsListDirectory(dfs->fs,path,&numEntries);
+
+ if (info) {
+ hdfsFreeFileInfo(info,numEntries);
+ }
+ st->st_nlink = numEntries + 2;
+ } else {
+ // not a directory
+ st->st_nlink = 1;
+ }
+
+ // set stat metadata
+ st->st_size = (info[0].mKind == kObjectKindDirectory) ? 4096 : info[0].mSize;
+ st->st_blksize = blksize;
+ st->st_blocks = ceil(st->st_size/st->st_blksize);
+ st->st_mode = (info[0].mKind == kObjectKindDirectory) ? (S_IFDIR | 0777) : (S_IFREG | 0666);
+ st->st_uid = default_id;
+ st->st_gid = default_id;
+ st->st_atime = info[0].mLastMod;
+ st->st_mtime = info[0].mLastMod;
+ st->st_ctime = info[0].mLastMod;
+
+ // free the info pointer
+ hdfsFreeFileInfo(info,1);
+
+ return 0;
+}
+
+static int dfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
+ off_t offset, struct fuse_file_info *fi)
+{
+ (void) offset;
+ (void) fi;
+
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(dfs);
+ assert(path);
+ assert(buf);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ int path_len = strlen(path);
+
+ // call dfs to read the dir
+ int numEntries = 0;
+ hdfsFileInfo *info = hdfsListDirectory(dfs->fs,path,&numEntries);
+
+ // NULL means either the directory doesn't exist or maybe IO error.
+ if (NULL == info) {
+ return -ENOENT;
+ }
+
+ int i ;
+ for (i = 0; i < numEntries; i++) {
+
+ // check the info[i] struct
+ if (NULL == info[i].mName) {
+ syslog(LOG_ERR,"ERROR: for <%s> info[%d].mName==NULL %s:%d", path, i, __FILE__,__LINE__);
+ continue;
+ }
+
+ struct stat st;
+ memset(&st, 0, sizeof(struct stat));
+
+ // set to 0 to indicate not supported for directory because we cannot (efficiently) get this info for every subdirectory
+ st.st_nlink = (info[i].mKind == kObjectKindDirectory) ? 0 : 1;
+
+ // setup stat size and acl meta data
+ st.st_size = info[i].mSize;
+ st.st_blksize = 512;
+ st.st_blocks = ceil(st.st_size/st.st_blksize);
+ st.st_mode = (info[i].mKind == kObjectKindDirectory) ? (S_IFDIR | 0777) : (S_IFREG | 0666);
+ st.st_uid = default_id;
+ st.st_gid = default_id;
+ st.st_atime = info[i].mLastMod;
+ st.st_mtime = info[i].mLastMod;
+ st.st_ctime = info[i].mLastMod;
+
+ // hack city: todo fix the below to something nicer and more maintainable but
+ // with good performance
+ // strip off the path but be careful if the path is solely '/'
+ // NOTE - this API started returning filenames as full dfs uris
+ const char *const str = info[i].mName + dfs->dfs_uri_len + path_len + ((path_len == 1 && *path == '/') ? 0 : 1);
+
+ // pack this entry into the fuse buffer
+ int res = 0;
+ if ((res = filler(buf,str,&st,0)) != 0) {
+ syslog(LOG_ERR, "ERROR: readdir filling the buffer %d %s:%d\n",res, __FILE__, __LINE__);
+ }
+
+ }
+
+ // insert '.' and '..'
+ const char *const dots [] = { ".",".."};
+ for (i = 0 ; i < 2 ; i++)
+ {
+ struct stat st;
+ memset(&st, 0, sizeof(struct stat));
+
+ // set to 0 to indicate not supported for directory because we cannot (efficiently) get this info for every subdirectory
+ st.st_nlink = 0;
+
+ // setup stat size and acl meta data
+ st.st_size = 512;
+ st.st_blksize = 512;
+ st.st_blocks = 1;
+ st.st_mode = (S_IFDIR | 0777);
+ st.st_uid = default_id;
+ st.st_gid = default_id;
+ // todo fix below times
+ st.st_atime = 0;
+ st.st_mtime = 0;
+ st.st_ctime = 0;
+
+ const char *const str = dots[i];
+
+ // flatten the info using fuse's function into a buffer
+ int res = 0;
+ if ((res = filler(buf,str,&st,0)) != 0) {
+ syslog(LOG_ERR, "ERROR: readdir filling the buffer %d %s:%d", res, __FILE__, __LINE__);
+ }
+ }
+
+ // free the info pointers
+ hdfsFreeFileInfo(info,numEntries);
+
+ return 0;
+}
+
+static int dfs_read(const char *path, char *buf, size_t size, off_t offset,
+ struct fuse_file_info *fi)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(dfs);
+ assert(path);
+ assert(buf);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+
+#ifdef OPTIMIZED_READS
+ dfs_fh *fh = (dfs_fh*)fi->fh;
+ //fprintf(stderr, "Cache bounds for %s: %llu -> %llu (%d bytes). Check for offset %llu\n", path, fh->startOffset, fh->startOffset + fh->sizeBuffer, fh->sizeBuffer, offset);
+ if (fh->sizeBuffer == 0 || offset < fh->startOffset || offset > (fh->startOffset + fh->sizeBuffer) )
+ {
+ // do the actual read
+ //fprintf (stderr,"Reading %s from HDFS, offset %llu, amount %d\n", path, offset, rd_cache_buf_size);
+ const tSize num_read = hdfsPread(dfs->fs, fh->hdfsFH, offset, fh->buf, rd_cache_buf_size);
+ if (num_read < 0) {
+ syslog(LOG_ERR, "Read error - pread failed for %s with return code %d %s:%d", path, num_read, __FILE__, __LINE__);
+ hdfsDisconnect(dfs->fs);
+ dfs->fs = NULL;
+ return -EIO;
+ }
+ fh->sizeBuffer = num_read;
+ fh->startOffset = offset;
+ //fprintf (stderr,"Read %d bytes of %s from HDFS\n", num_read, path);
+ }
+
+ char* local_buf = fh->buf;
+ const tSize cacheLookupOffset = offset - fh->startOffset;
+ local_buf += cacheLookupOffset;
+ //fprintf(stderr,"FUSE requested %d bytes of %s for offset %d in file\n", size, path, offset);
+ const tSize amount = cacheLookupOffset + size > fh->sizeBuffer
+ ? fh->sizeBuffer - cacheLookupOffset
+ : size;
+ //fprintf(stderr,"Reading %s from cache, %d bytes from position %d\n", path, amount, cacheLookupOffset);
+ //fprintf(stderr,"Cache status for %s: %d bytes cached from offset %llu\n", path, fh->sizeBuffer, fh->startOffset);
+ memcpy(buf, local_buf, amount);
+ //fprintf(stderr,"Read %s from cache, %d bytes from position %d\n", path, amount, cacheLookupOffset);
+ //fprintf(stderr,"Cache status for %s: %d bytes cached from offset %llu\n", path, fh->sizeBuffer, fh->startOffset);
+ return amount;
+
+#else
+ // NULL means either file doesn't exist or maybe IO error - i.e., the dfs_open must have failed
+ if (NULL == (void*)fi->fh) {
+ // should never happen
+ return -EIO;
+ }
+ syslog(LOG_DEBUG,"buffer size=%d\n",(int)size);
+
+ // do the actual read
+ const tSize num_read = hdfsPread(dfs->fs, (hdfsFile)fi->fh, offset, buf, size);
+
+ // handle errors
+ if (num_read < 0) {
+ syslog(LOG_ERR, "Read error - pread failed for %s with return code %d %s:%d", path, num_read, __FILE__, __LINE__);
+ hdfsDisconnect(dfs->fs);
+ dfs->fs = NULL;
+ return -EIO;
+ }
+ return num_read;
+#endif
+
+}
+
+static int dfs_statfs(const char *path, struct statvfs *st)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(path);
+ assert(st);
+ assert(dfs);
+
+ // init the stat structure
+ memset(st,0,sizeof(struct statvfs));
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ const long cap = hdfsGetCapacity(dfs->fs);
+ const long used = hdfsGetUsed(dfs->fs);
+ const long bsize = hdfsGetDefaultBlockSize(dfs->fs);
+
+ // fill in the statvfs structure
+
+ /* FOR REFERENCE:
+ struct statvfs {
+ unsigned long f_bsize; // file system block size
+ unsigned long f_frsize; // fragment size
+ fsblkcnt_t f_blocks; // size of fs in f_frsize units
+ fsblkcnt_t f_bfree; // # free blocks
+ fsblkcnt_t f_bavail; // # free blocks for non-root
+ fsfilcnt_t f_files; // # inodes
+ fsfilcnt_t f_ffree; // # free inodes
+ fsfilcnt_t f_favail; // # free inodes for non-root
+ unsigned long f_fsid; // file system id
+ unsigned long f_flag; / mount flags
+ unsigned long f_namemax; // maximum filename length
+ };
+ */
+
+ st->f_bsize = bsize;
+ st->f_frsize = st->f_bsize;
+ st->f_blocks = cap/st->f_bsize;
+ st->f_bfree = (cap-used)/st->f_bsize;
+ st->f_bavail = st->f_bfree;
+ st->f_files = 1000;
+ st->f_ffree = 500;
+ st->f_favail = 500;
+ st->f_fsid = 1023;
+ st->f_flag = ST_RDONLY | ST_NOSUID;
+ st->f_namemax = 1023;
+
+ return 0;
+}
+
+static int dfs_access(const char *path, int mask)
+{
+ // no permissions on dfs, always a success
+ return 0;
+}
+
+//
+// The remainder are write functionality and therefore not implemented right now
+//
+
+
+static char **protectedpaths;
+
+
+static int dfs_mkdir(const char *path, mode_t mode)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(path);
+ assert(dfs);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ assert('/' == *path);
+
+ int i ;
+ for (i = 0; protectedpaths[i]; i++) {
+ if (strcmp(path, protectedpaths[i]) == 0) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to create the directory: %s", path);
+ return -EACCES;
+ }
+ }
+
+
+ if (dfs->nowrites || hdfsCreateDirectory(dfs->fs, path)) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to create directory %s",path);
+ return -EIO;
+ }
+
+ return 0;
+
+}
+
+static int dfs_rename(const char *from, const char *to)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(from);
+ assert(to);
+ assert(dfs);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ assert('/' == *from);
+ assert('/' == *to);
+
+ int i ;
+ for (i = 0; protectedpaths[i] != NULL; i++) {
+ if (strcmp(from, protectedpaths[i]) == 0) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to rename directories %s to %s",from,to);
+ return -EACCES;
+ }
+ if (strcmp(to, protectedpaths[i]) == 0) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to rename directories %s to %s",from,to);
+ return -EACCES;
+ }
+ }
+
+ if (dfs->nowrites || hdfsRename(dfs->fs, from, to)) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to rename %s to %s",from, to);
+ return -EIO;
+ }
+ return 0;
+
+}
+
+
+static int dfs_rmdir(const char *path)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(path);
+ assert(dfs);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ assert('/' == *path);
+
+ int i ;
+ for (i = 0; protectedpaths[i]; i++) {
+ if (strcmp(path, protectedpaths[i]) == 0) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to delete the directory: %s ",path);
+ return -EACCES;
+ }
+ }
+
+ int numEntries = 0;
+ hdfsFileInfo *info = hdfsListDirectory(dfs->fs,path,&numEntries);
+
+ // free the info pointers
+ hdfsFreeFileInfo(info,numEntries);
+
+ if (numEntries) {
+ return -ENOTEMPTY;
+ }
+
+
+
+ // since these commands go through the programmatic hadoop API, there is no
+ // trash feature. So, force it here.
+ // But make sure the person isn't deleting from Trash itself :)
+ // NOTE: /Trash is in protectedpaths so they cannot delete all of trash
+ if (!dfs->no_trash && strncmp(path, "/Trash", strlen("/Trash")) != 0) {
+
+ char target[4096];
+ char dir[4096];
+ int status;
+
+ {
+ // find the directory and full targets in Trash
+
+ sprintf(target, "/Trash/Current%s",path);
+
+ // strip off the actual file or directory name from the fullpath
+ char *name = rindex(path, '/');
+ assert(name);
+ *name = 0;
+
+ // use that path to ensure the directory exists in the Trash dir
+ // prepend Trash to the directory
+ sprintf(dir,"/Trash/Current%s/",path);
+
+ // repair the path not used again but in case the caller expects it.
+ *name = '/';
+ }
+
+ // if the directory doesn't already exist in the Trash
+ // then we go through with the rename
+ if ( hdfsExists(dfs->fs, target) != 0) { // 0 means it exists. weird
+ // make the directory to put it in in the Trash
+ if ((status = dfs_mkdir(dir,0)) != 0) {
+ return status;
+ }
+
+ // do the rename
+ return dfs_rename(path,target);
+
+ }
+ // if the directory exists in the Trash, then we don't bother doing the rename
+ // and just delete the existing one by falling though.
+ }
+
+ if (dfs->nowrites || hdfsDelete(dfs->fs, path)) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to delete the directory %s",path);
+ return -EIO;
+ }
+ return 0;
+}
+
+
+static int dfs_unlink(const char *path)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(path);
+ assert(dfs);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ assert('/' == *path);
+
+ int i ;
+ for (i = 0; protectedpaths[i]; i++) {
+ if (strcmp(path, protectedpaths[i]) == 0) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to delete the directory: %s ",path);
+ return -EACCES;
+ }
+ }
+
+
+
+ // since these commands go through the programmatic hadoop API, there is no
+ // trash feature. So, force it here.
+ // But make sure the person isn't deleting from Trash itself :)
+ // NOTE: /Trash is in protectedpaths so they cannot delete all of trash
+ if (!dfs->no_trash && strncmp(path, "/Trash", strlen("/Trash")) != 0) {
+
+ char target[4096];
+ char dir[4096];
+ int status;
+
+ {
+ // find the directory and full targets in Trash
+
+ sprintf(target, "/Trash/Current%s",path);
+
+ // strip off the actual file or directory name from the fullpath
+ char *name = rindex(path, '/');
+ assert(name);
+ *name = 0;
+
+ // use that path to ensure the directory exists in the Trash dir
+ // prepend Trash to the directory
+ sprintf(dir,"/Trash/Current%s/",path);
+
+ // repair the path not used again but in case the caller expects it.
+ *name = '/';
+ }
+
+ // if this is a file and it's already got a copy in the Trash, to be conservative, we
+ // don't do the delete.
+ if (hdfsExists(dfs->fs, target) == 0) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to delete a file that was already deleted so cannot back it to Trash: %s",target);
+ return -EIO;
+ }
+
+ // make the directory to put it in in the Trash
+ if ((status = dfs_mkdir(dir,0)) != 0) {
+ return status;
+ }
+
+ // do the rename
+ return dfs_rename(path,target);
+ }
+
+ if (dfs->nowrites || hdfsDelete(dfs->fs, path)) {
+ syslog(LOG_ERR,"ERROR: hdfs trying to delete the file %s",path);
+ return -EIO;
+ }
+ return 0;
+
+}
+
+static int dfs_chmod(const char *path, mode_t mode)
+{
+ (void)path;
+ (void)mode;
+ return -ENOTSUP;
+}
+
+static int dfs_chown(const char *path, uid_t uid, gid_t gid)
+{
+ (void)path;
+ (void)uid;
+ (void)gid;
+ return -ENOTSUP;
+}
+
+static int dfs_truncate(const char *path, off_t size)
+{
+ (void)path;
+ (void)size;
+ return -ENOTSUP;
+}
+
+long tempfh = 0;
+
+static int dfs_open(const char *path, struct fuse_file_info *fi)
+{
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(path);
+ assert('/' == *path);
+ assert(dfs);
+
+ int ret = 0;
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ // 0x8000 is always passed in and hadoop doesn't like it, so killing it here
+ // bugbug figure out what this flag is and report problem to Hadoop JIRA
+ int flags = (fi->flags & 0x7FFF);
+
+#ifdef OPTIMIZED_READS
+ // retrieve dfs specific data
+ dfs_fh *fh = (dfs_fh*)malloc(sizeof (dfs_fh));
+ fi->fh = (uint64_t)fh;
+ fh->hdfsFH = (hdfsFile)hdfsOpenFile(dfs->fs, path, flags, 0, 3, 0);
+ fh->buf = (char*)malloc(rd_cache_buf_size*sizeof (char));
+ fh->startOffset = 0;
+ fh->sizeBuffer = 0;
+
+ if (0 == fh->hdfsFH) {
+ syslog(LOG_ERR, "ERROR: could not open file %s dfs %s:%d\n", path,__FILE__, __LINE__);
+ ret = -EIO;
+ }
+#else
+
+ // retrieve dfs specific data
+ fi->fh = (uint64_t)hdfsOpenFile(dfs->fs, path, flags, 0, 3, 0);
+
+ if (0 == fi->fh) {
+ syslog(LOG_ERR, "ERROR: could not open file %s dfs %s:%d\n", path,__FILE__, __LINE__);
+ ret = -EIO;
+ }
+
+#endif
+
+ return ret;
+}
+
+static int dfs_write(const char *path, const char *buf, size_t size,
+ off_t offset, struct fuse_file_info *fi)
+{
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(path);
+ assert(dfs);
+ assert('/' == *path);
+
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+#ifdef OPTIMIZED_READS
+
+ dfs_fh *fh = (dfs_fh*)fi->fh;
+ hdfsFile file_handle = (hdfsFile)fh->hdfsFH;
+
+#else
+ hdfsFile file_handle = (hdfsFile)fi->fh;
+
+ if (NULL == file_handle) {
+ syslog(LOG_ERR, "ERROR: fuse problem - no file_handle for %s %s:%d\n",path, __FILE__, __LINE__);
+ return -EIO;
+ }
+#endif
+
+ // syslog(LOG_DEBUG,"hdfsTell(dfs,%ld)\n",(long)file_handle);
+// tOffset cur_offset = hdfsTell(dfs->fs, file_handle);
+
+ // if (cur_offset != offset) {
+ // syslog(LOG_ERR, "ERROR: user trying to random access write to a file %d!=%d for %s %s:%d\n",(int)cur_offset, (int)offset,path, __FILE__, __LINE__);
+// return -EIO;
+// }
+
+
+ syslog(LOG_DEBUG,"hdfsWrite(dfs,%ld,'%s',%d)\n",(long)file_handle,buf,(int)size);
+ tSize length = hdfsWrite(dfs->fs, file_handle, buf, size);
+
+
+ if (length != size) {
+ syslog(LOG_ERR, "ERROR: fuse problem - could not write all the bytes for %s %d!=%d%s:%d\n",path,length,(int)size, __FILE__, __LINE__);
+ return -EIO;
+ }
+ return 0;
+
+}
+
+int dfs_release (const char *path, struct fuse_file_info *fi) {
+
+ // retrieve dfs specific data
+ dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data;
+
+ // check params and the context var
+ assert(path);
+ assert(dfs);
+ assert('/' == *path);
+ // if not connected, try to connect and fail out if we can't.
+ if (NULL == dfs->fs && NULL == (dfs->fs = hdfsConnect(dfs->nn_hostname,dfs->nn_port))) {
+ syslog(LOG_ERR, "ERROR: could not connect to dfs %s:%d\n", __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ if (NULL == (void*)fi->fh) {
+ return 0;
+ }
+
+#ifdef OPTIMIZED_READS
+ dfs_fh *fh = (dfs_fh*)fi->fh;
+ hdfsFile file_handle = (hdfsFile)fh->hdfsFH;
+ free(fh->buf);
+ free(fh);
+
+#else
+ hdfsFile file_handle = (hdfsFile)fi->fh;
+#endif
+
+ if (NULL == file_handle) {
+ return 0;
+ }
+
+ if (hdfsCloseFile(dfs->fs, file_handle) != 0) {
+ syslog(LOG_ERR, "ERROR: dfs problem - could not close file_handle for %s %s:%d\n",path, __FILE__, __LINE__);
+ return -EIO;
+ }
+
+ fi->fh = (uint64_t)0;
+ return 0;
+}
+
+static int dfs_mknod(const char *path, mode_t mode, dev_t rdev) {
+ syslog(LOG_DEBUG,"in dfs_mknod");
+ return 0;
+}
+
+static int dfs_create(const char *path, mode_t mode, struct fuse_file_info *fi)
+{
+ syslog(LOG_DEBUG,"in dfs_create");
+ fi->flags |= mode;
+
+ return dfs_open(path, fi);
+}
+int dfs_flush(const char *path, struct fuse_file_info *fi) {
+ return 0;
+}
+
+
+void dfs_setattr(struct stat *attr, int to_set, struct fuse_file_info *fi)
+{
+
+}
+
+void dfs_destroy (void *ptr)
+{
+ dfs_context *dfs = (dfs_context*)ptr;
+ hdfsDisconnect(dfs->fs);
+ dfs->fs = NULL;
+}
+
+
+// Hacked up function to basically do:
+// protectedpaths = split(PROTECTED_PATHS,',');
+
+static void init_protectedpaths() {
+ // PROTECTED_PATHS should be a #defined value from autoconf
+ // set it with configure --with-protectedpaths=/,/user,/user/foo
+ // note , seped with no other spaces and no quotes around it
+ char *tmp = PROTECTED_PATHS;
+
+ assert(tmp);
+
+ // handle degenerate case up front.
+ if (0 == *tmp) {
+ protectedpaths = (char**)malloc(sizeof(char*));
+ protectedpaths[0] = NULL;
+ return;
+ }
+
+ int i = 0;
+ while (tmp && (NULL != (tmp = index(tmp,',')))) {
+ tmp++; // pass the ,
+ i++;
+ }
+ i++; // for the last entry
+ i++; // for the final NULL
+ protectedpaths = (char**)malloc(sizeof(char*)*i);
+ printf("i=%d\n",i);
+ tmp = PROTECTED_PATHS;
+ int j = 0;
+ while (NULL != tmp && j < i) {
+ int length;
+ char *eos = index(tmp,',');
+ if (NULL != eos) {
+ length = eos - tmp; // length of this value
+ } else {
+ length = strlen(tmp);
+ }
+ protectedpaths[j] = (char*)malloc(sizeof(char)*length+1);
+ strncpy(protectedpaths[j], tmp, length);
+ protectedpaths[j][length] = '\0';
+ if (eos) {
+ tmp = eos + 1;
+ } else {
+ tmp = NULL;
+ }
+ j++;
+ }
+ protectedpaths[j] = NULL;
+ /*
+ j = 0;
+ while (protectedpaths[j]) {
+ printf("protectedpaths[%d]=%s\n",j,protectedpaths[j]);
+ fflush(stdout);
+ j++;
+ }
+ exit(1);
+ */
+}
+
+
+
+void *dfs_init()
+{
+
+ //
+ // Create a private struct of data we will pass to fuse here and which
+ // will then be accessible on every call.
+ //
+ dfs_context *dfs = (dfs_context*)malloc(sizeof (dfs_context));
+
+ if (NULL == dfs) {
+ syslog(LOG_ERR, "FATAL: could not malloc fuse dfs context struct - out of memory %s:%d", __FILE__, __LINE__);
+ exit(1);
+ }
+
+ // initialize the context
+ dfs->debug = options.debug;
+ dfs->nn_hostname = options.server;
+ dfs->nn_port = options.port;
+ dfs->fs = NULL;
+ dfs->nowrites = options.nowrites;
+ dfs->no_trash = options.no_trash;
+
+ bzero(dfs->dfs_uri,0);
+ sprintf(dfs->dfs_uri,"dfs://%s:%d/",dfs->nn_hostname,dfs->nn_port);
+ dfs->dfs_uri_len = strlen(dfs->dfs_uri);
+
+ // use ERR level to ensure it makes it into the log.
+ syslog(LOG_ERR, "mounting %s", dfs->dfs_uri);
+
+ init_protectedpaths();
+
+ return (void*)dfs;
+}
+
+
+static struct fuse_operations dfs_oper = {
+ .getattr = dfs_getattr,
+ .access = dfs_access,
+ .readdir = dfs_readdir,
+ .destroy = dfs_destroy,
+ .init = dfs_init,
+ .open = dfs_open,
+ .read = dfs_read,
+ .statfs = dfs_statfs,
+ .mkdir = dfs_mkdir,
+ .rmdir = dfs_rmdir,
+ .rename = dfs_rename,
+ .unlink = dfs_unlink,
+ .release = dfs_release,
+ // .create = dfs_create,
+ // .write = dfs_write,
+ // .flush = dfs_flush,
+ //.xsetattr = dfs_setattr,
+ // .mknod = dfs_mknod,
+ .chmod = dfs_chmod,
+ .chown = dfs_chown,
+ // .truncate = dfs_truncate,
+};
+
+
+int main(int argc, char *argv[])
+{
+ umask(0);
+
+ program = argv[0];
+ struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
+
+ /* clear structure that holds our options */
+ memset(&options, 0, sizeof(struct options));
+
+ if (fuse_opt_parse(&args, &options, dfs_opts, dfs_options) == -1)
+ /** error parsing options */
+ return -1;
+
+ if (options.server == NULL || options.port == 0) {
+ print_usage(argv[0]);
+ exit(0);
+ }
+ int ret = fuse_main(args.argc, args.argv, &dfs_oper, NULL);
+
+ if (ret) printf("\n");
+
+ /** free arguments */
+ fuse_opt_free_args(&args);
+
+ return ret;
+}
Added: hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh Thu May 29 13:18:11 2008
@@ -0,0 +1,37 @@
+#
+# Copyright 2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ "$HADOOP_HOME" = "" ]; then
+ HADOOP_HOME=/usr/local/share/hadoop
+fi
+
+for f in ls $HADOOP_HOME/lib/*.jar $HADOOP_HOME/*.jar ; do
+ CLASSPATH=$CLASSPATH:$f
+done
+
+if [ "$OS_ARCH" = "" ]; then
+ OS_ARCH=amd64
+fi
+
+if [ "$JAVA_HOME" = "" ]; then
+ JAVA_HOME=/usr/local/java
+fi
+
+if [ "$LD_LIBRARY_PATH" = "" ]; then
+ LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/$OS_ARCH/server:/usr/local/share/hdfs/libhdfs/:/usr/local/lib
+fi
+
+./fuse_dfs $@ -o-o allow_other
Propchange: hadoop/core/trunk/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/core/trunk/src/contrib/fuse-dfs/test/TestFuseDFS.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fuse-dfs/test/TestFuseDFS.java?rev=661462&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/fuse-dfs/test/TestFuseDFS.java (added)
+++ hadoop/core/trunk/src/contrib/fuse-dfs/test/TestFuseDFS.java Thu May 29 13:18:11 2008
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.hadoop.dfs.*;
+import junit.framework.TestCase;
+import java.io.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
+import java.net.*;
+
+/**
+ * This class tests that the Fuse module for DFS can mount properly
+ * and does a few simple commands:
+ * mkdir
+ * rmdir
+ * ls
+ * cat
+ *
+ * cp and touch are purposely not tested because they won't work with the current module
+
+ *
+ */
+public class TestFuseDFS extends TestCase {
+
+ /**
+ * mount the fuse file system using assumed fuse module library installed in /usr/local/lib or somewhere else on your
+ * pre-existing LD_LIBRARY_PATH
+ *
+ */
+ private void mount(String mountpoint, URI dfs) throws IOException, InterruptedException {
+ String cp = System.getenv("CLASSPATH");
+ String libhdfs = "../../c++/libhdfs/";
+ String lp = System.getenv("LD_LIBRARY_PATH") + ":" + "/usr/local/lib:" + libhdfs;
+
+ Runtime r = Runtime.getRuntime();
+ String fuse_cmd = "../src/fuse_dfs";
+
+ String cmd = fuse_cmd;
+ cmd += " ";
+ // cmd += dfs.toASCIIString();
+ cmd += "dfs://";
+ cmd += dfs.getHost();
+ cmd += ":" ;
+ cmd += String.valueOf(dfs.getPort());
+ cmd += " ";
+ cmd += mountpoint;
+ final String [] envp = {
+ "CLASSPATH="+ cp,
+ "LD_LIBRARY_PATH=" + lp
+ };
+
+
+ // ensure the mount point is not currently mounted
+ Process p = r.exec("sudo umount -l " + mountpoint);
+ p.waitFor();
+
+ // make the mount point if needed
+ p = r.exec("mkdir -p " + mountpoint);
+ assertTrue(p.waitFor() == 0);
+
+ System.err.println("cmd=" + cmd);
+ // mount fuse to the mount point
+ p = r.exec(cmd, envp);
+ assertTrue(p.waitFor() == 0);
+ assertTrue(p.exitValue() == 0);
+ }
+
+ /**
+ * unmounts fuse for before shutting down.
+ */
+ private void umount(String mpoint) throws IOException, InterruptedException {
+ Runtime r= Runtime.getRuntime();
+ Process p = r.exec("sudo umount -l " + mpoint);
+ p.waitFor();
+ }
+
+ /**
+ * Set things up - create mini dfs cluster and mount the fuse filesystem.
+ */
+ public TestFuseDFS() throws IOException,InterruptedException {
+ Configuration conf = new Configuration();
+ this.cluster = new MiniDFSCluster(conf, 1, true, null);
+ this.fileSys = this.cluster.getFileSystem();
+ String mpoint = "/tmp/testfuse";
+ this.mount(mpoint, fileSys.getUri());
+ this.myPath = new Path("/test/mkdirs");
+ }
+
+ private MiniDFSCluster cluster;
+ private FileSystem fileSys;
+ private String mpoint = "/tmp/testfuse";
+ private Path myPath;
+
+
+ /**
+ * use shell to create a dir and then use filesys to see it exists.
+ */
+ public void testMkdir() throws IOException,InterruptedException {
+ // First create a new directory with mkdirs
+ Runtime r = Runtime.getRuntime();
+ Process p = r.exec("mkdir -p " + mpoint + "/test/mkdirs");
+ assertTrue(p.waitFor() == 0);
+ assertTrue(p.exitValue() == 0);
+
+ assertTrue(this.fileSys.exists(myPath));
+
+
+ }
+
+ /**
+ * Test ls for dir already created in testMkdDir also tests bad ls
+ */
+ public void testLs() throws IOException,InterruptedException {
+ // First create a new directory with mkdirs
+ Runtime r = Runtime.getRuntime();
+ Process p = r.exec("ls " + mpoint + "/test/mkdirs");
+ assertTrue(p.waitFor() == 0);
+ assertTrue(p.exitValue() == 0);
+
+ p = r.exec("ls " + mpoint + "/test/mkdirsNotThere");
+ assertFalse(p.waitFor() == 0);
+ assertFalse(p.exitValue() == 0);
+
+ }
+
+ /**
+ * Remove a dir using the shell and use filesys to see it no longer exists.
+ */
+ public void testRmdir() throws IOException,InterruptedException {
+ // First create a new directory with mkdirs
+ Path myPath = new Path("/test/mkdirs");
+ assertTrue(fileSys.exists(myPath));
+
+ Runtime r = Runtime.getRuntime();
+ Process p = r.exec("rmdir " + mpoint + "/test/mkdirs");
+
+ assertTrue(p.waitFor() == 0);
+ assertTrue(p.exitValue() == 0);
+
+ assertFalse(fileSys.exists(myPath));
+ }
+
+
+ /**
+ * Use filesys to create the hello world! file and then cat it and see its contents are correct.
+ */
+ public void testCat() throws IOException,InterruptedException {
+ // First create a new directory with mkdirs
+
+ Path myPath = new Path("/test/hello");
+ FSDataOutputStream s = fileSys.create(myPath);
+ String hello = "hello world!";
+
+ s.write(hello.getBytes());
+ s.close();
+
+ assertTrue(fileSys.exists(myPath));
+
+ Runtime r = Runtime.getRuntime();
+ Process p = r.exec("cat " + mpoint + "/test/hello");
+
+ assertTrue(p.waitFor() == 0);
+ assertTrue(p.exitValue() == 0);
+ InputStream i = p.getInputStream();
+ byte b[] = new byte[1024];
+ int length = i.read(b);
+ String s2 = new String(b,0,length);
+ assertTrue(s2.equals(hello));
+ }
+
+
+ /**
+ * Unmount and close
+ */
+ public void finalize() {
+ try {
+ this.close();
+ } catch(Exception e) { }
+ }
+
+ /**
+ * Unmount and close
+ */
+ public void close() throws IOException, InterruptedException {
+ this.umount(mpoint);
+ if(this.fileSys != null) {
+ this.fileSys.close();
+ this.fileSys = null;
+ }
+ if(this.cluster != null) {
+ this.cluster.shutdown();
+ this.cluster = null;
+ }
+ }
+
+ public static void main(String args[]) {
+ try {
+ TestFuseDFS d = new TestFuseDFS();
+ d.testMkdir();
+ d.testLs();
+ d.testRmdir();
+ d.testCat();
+ d.close();
+ } catch(Exception e) {
+ System.err.println("e=" + e.getMessage());
+ e.printStackTrace();
+ }
+
+ }
+}