You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2008/08/29 06:32:00 UTC
svn commit: r690096 [1/15] - in /hadoop/core/trunk: ./ src/contrib/thriftfs/
src/contrib/thriftfs/gen-cocoa/ src/contrib/thriftfs/gen-cpp/
src/contrib/thriftfs/gen-java/ src/contrib/thriftfs/gen-java/org/
src/contrib/thriftfs/gen-java/org/apache/ src/c...
Author: omalley
Date: Thu Aug 28 21:31:57 2008
New Revision: 690096
URL: http://svn.apache.org/viewvc?rev=690096&view=rev
Log:
HADOOP-3754. Add a thrift interface to access HDFS. (dhruba via omalley)
Added:
hadoop/core/trunk/src/contrib/thriftfs/
hadoop/core/trunk/src/contrib/thriftfs/README
hadoop/core/trunk/src/contrib/thriftfs/build.xml
hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/
hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h
hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.m
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem.cpp
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem.h
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem_server.skeleton.cpp
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_constants.cpp
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_constants.h
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_types.cpp
hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_types.h
hadoop/core/trunk/src/contrib/thriftfs/gen-java/
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/BlockLocation.java
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/FileStatus.java
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/MalformedInputException.java
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/Pathname.java
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftHadoopFileSystem.java
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftHandle.java
hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftIOException.java
hadoop/core/trunk/src/contrib/thriftfs/gen-perl/
hadoop/core/trunk/src/contrib/thriftfs/gen-perl/Constants.pm
hadoop/core/trunk/src/contrib/thriftfs/gen-perl/ThriftHadoopFileSystem.pm
hadoop/core/trunk/src/contrib/thriftfs/gen-perl/Types.pm
hadoop/core/trunk/src/contrib/thriftfs/gen-php/
hadoop/core/trunk/src/contrib/thriftfs/gen-php/ThriftHadoopFileSystem.php
hadoop/core/trunk/src/contrib/thriftfs/gen-php/hadoopfs_types.php
hadoop/core/trunk/src/contrib/thriftfs/gen-py/
hadoop/core/trunk/src/contrib/thriftfs/gen-py/__init__.py
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem-remote (with props)
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem.py
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem.pyc (with props)
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/__init__.py
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/__init__.pyc (with props)
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/constants.py
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ttypes.py
hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ttypes.pyc (with props)
hadoop/core/trunk/src/contrib/thriftfs/gen-rb/
hadoop/core/trunk/src/contrib/thriftfs/gen-rb/ThriftHadoopFileSystem.rb
hadoop/core/trunk/src/contrib/thriftfs/gen-rb/hadoopfs_constants.rb
hadoop/core/trunk/src/contrib/thriftfs/gen-rb/hadoopfs_types.rb
hadoop/core/trunk/src/contrib/thriftfs/gen-st/
hadoop/core/trunk/src/contrib/thriftfs/gen-st/hadoopfs.st
hadoop/core/trunk/src/contrib/thriftfs/if/
hadoop/core/trunk/src/contrib/thriftfs/if/hadoopfs.thrift
hadoop/core/trunk/src/contrib/thriftfs/lib/
hadoop/core/trunk/src/contrib/thriftfs/lib/README
hadoop/core/trunk/src/contrib/thriftfs/lib/Thrift.LICENSE
hadoop/core/trunk/src/contrib/thriftfs/lib/hadoopthriftapi.jar (with props)
hadoop/core/trunk/src/contrib/thriftfs/lib/libthrift.jar (with props)
hadoop/core/trunk/src/contrib/thriftfs/scripts/
hadoop/core/trunk/src/contrib/thriftfs/scripts/hdfs.py (with props)
hadoop/core/trunk/src/contrib/thriftfs/scripts/start_thrift_server.sh (with props)
hadoop/core/trunk/src/contrib/thriftfs/src/
hadoop/core/trunk/src/contrib/thriftfs/src/java/
hadoop/core/trunk/src/contrib/thriftfs/src/java/org/
hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/
hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/
hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/thriftfs/
hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/thriftfs/HadoopThriftServer.java
hadoop/core/trunk/src/contrib/thriftfs/test/
hadoop/core/trunk/src/contrib/thriftfs/test/org/
hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/
hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/
hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/thriftfs/
hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/thriftfs/TestThriftfs.java
Modified:
hadoop/core/trunk/CHANGES.txt
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=690096&r1=690095&r2=690096&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Aug 28 21:31:57 2008
@@ -106,6 +106,8 @@
HADOOP-3746. Add a fair share scheduler. (Matei Zaharia via omalley)
+ HADOOP-3754. Add a thrift interface to access HDFS. (dhruba via omalley)
+
IMPROVEMENTS
HADOOP-3908. Fuse-dfs: better error message if llibhdfs.so doesn't exist.
Added: hadoop/core/trunk/src/contrib/thriftfs/README
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/README?rev=690096&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/thriftfs/README (added)
+++ hadoop/core/trunk/src/contrib/thriftfs/README Thu Aug 28 21:31:57 2008
@@ -0,0 +1,39 @@
+ Thrift API for HDFS
+ ==================
+
+Introduction:
+============
+
+The Hadoop Distributed File System is written in Java. An application
+that wants to store/fetch data to/from HDFS can use the Java API
+This means that applications that are not written in Java cannot
+access HDFS in an elegant manner.
+
+Thrift is a software framework for scalable cross-language services
+development. It combines a powerful software stack with a code generation
+engine to build services that work efficiently and seamlessly
+between C++, Java, Python, PHP, and Ruby.
+
+This project exposes HDFS APIs using the Thrift software stack. This
+allows applciations written in a myriad of languages to access
+HDFS elegantly.
+
+
+The Application Programming Interface (API)
+===========================================
+The HDFS API that is exposed through Thrift can be found in if/hadoopfs.thrift.
+
+Compilation
+===========
+The compilation process creates a server org.apache.hadoop.thriftfs.HadooopThriftServer
+that implements the Thrift interface defined in if/hadoopfs.thrift.
+
+Th thrift compiler is used to generate API stubs in python, php, ruby,
+cocoa, etc. The generated code is checked into the directories gen-*.
+The generated java API is checked into lib/hadoopthriftapi.jar.
+
+There is a sample python script hdfs.py in the scripts directory. This python
+script, when invoked, creates a HadoopThriftServer in the background, and then
+communicates wth HDFS using the API. This script is for demonstration purposes
+only.
+
Added: hadoop/core/trunk/src/contrib/thriftfs/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/build.xml?rev=690096&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/thriftfs/build.xml (added)
+++ hadoop/core/trunk/src/contrib/thriftfs/build.xml Thu Aug 28 21:31:57 2008
@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+Before you can run these subtargets directly, you need
+to call at top-level: ant deploy-contrib compile-core-test
+-->
+<project name="thriftfs" default="jar">
+
+ <import file="../build-contrib.xml"/>
+
+ <!-- create the list of files to add to the classpath -->
+ <fileset dir="${hadoop.root}/lib" id="class.path">
+ <include name="**/*.jar" />
+ <exclude name="**/excluded/" />
+ </fileset>
+
+ <!-- Override jar target to specify main class -->
+ <target name="jar" depends="compile">
+ <jar
+ jarfile="${build.dir}/hadoop-${version}-${name}.jar"
+ basedir="${build.classes}"
+ >
+ <manifest>
+ <attribute name="Main-Class" value="org.apache.hadoop.thriftfs.HadooopThriftServer"/>
+ </manifest>
+ </jar>
+ </target>
+
+ <!-- Run only pure-Java unit tests. superdottest -->
+ <target name="test">
+ <antcall target="hadoopbuildcontrib.test">
+ <param name="test.exclude" value="TestStreamedMerge"/>
+ </antcall>
+ </target>
+
+ <!-- Run all unit tests
+ This is not called as part of the nightly build
+ because it will only run on platforms that have standard
+ Unix utilities available.
+ -->
+ <target name="test-unix">
+ <antcall target="hadoopbuildcontrib.test">
+ </antcall>
+ </target>
+
+
+</project>
Added: hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h?rev=690096&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h (added)
+++ hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h Thu Aug 28 21:31:57 2008
@@ -0,0 +1,214 @@
+/**
+ * Autogenerated by Thrift
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+
+#import <Cocoa/Cocoa.h>
+
+#import <TProtocol.h>
+#import <TApplicationException.h>
+#import <TProtocolUtil.h>
+
+
+@interface ThriftHandle : NSObject {
+ int64_t __id;
+
+ BOOL __id_isset;
+}
+
+- (id) initWithId: (int64_t) id;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (int64_t) id;
+- (void) setId: (int64_t) id;
+- (BOOL) idIsSet;
+
+@end
+
+@interface Pathname : NSObject {
+ NSString * __pathname;
+
+ BOOL __pathname_isset;
+}
+
+- (id) initWithPathname: (NSString *) pathname;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) pathname;
+- (void) setPathname: (NSString *) pathname;
+- (BOOL) pathnameIsSet;
+
+@end
+
+@interface FileStatus : NSObject {
+ NSString * __path;
+ int64_t __length;
+ BOOL __isdir;
+ int16_t __block_replication;
+ int64_t __blocksize;
+ int64_t __modification_time;
+ NSString * __permission;
+ NSString * __owner;
+ NSString * __group;
+
+ BOOL __path_isset;
+ BOOL __length_isset;
+ BOOL __isdir_isset;
+ BOOL __block_replication_isset;
+ BOOL __blocksize_isset;
+ BOOL __modification_time_isset;
+ BOOL __permission_isset;
+ BOOL __owner_isset;
+ BOOL __group_isset;
+}
+
+- (id) initWithPath: (NSString *) path length: (int64_t) length isdir: (BOOL) isdir block_replication: (int16_t) block_replication blocksize: (int64_t) blocksize modification_time: (int64_t) modification_time permission: (NSString *) permission owner: (NSString *) owner group: (NSString *) group;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) path;
+- (void) setPath: (NSString *) path;
+- (BOOL) pathIsSet;
+
+- (int64_t) length;
+- (void) setLength: (int64_t) length;
+- (BOOL) lengthIsSet;
+
+- (BOOL) isdir;
+- (void) setIsdir: (BOOL) isdir;
+- (BOOL) isdirIsSet;
+
+- (int16_t) block_replication;
+- (void) setBlock_replication: (int16_t) block_replication;
+- (BOOL) block_replicationIsSet;
+
+- (int64_t) blocksize;
+- (void) setBlocksize: (int64_t) blocksize;
+- (BOOL) blocksizeIsSet;
+
+- (int64_t) modification_time;
+- (void) setModification_time: (int64_t) modification_time;
+- (BOOL) modification_timeIsSet;
+
+- (NSString *) permission;
+- (void) setPermission: (NSString *) permission;
+- (BOOL) permissionIsSet;
+
+- (NSString *) owner;
+- (void) setOwner: (NSString *) owner;
+- (BOOL) ownerIsSet;
+
+- (NSString *) group;
+- (void) setGroup: (NSString *) group;
+- (BOOL) groupIsSet;
+
+@end
+
+@interface BlockLocation : NSObject {
+ NSArray * __hosts;
+ NSArray * __names;
+ int64_t __offset;
+ int64_t __length;
+
+ BOOL __hosts_isset;
+ BOOL __names_isset;
+ BOOL __offset_isset;
+ BOOL __length_isset;
+}
+
+- (id) initWithHosts: (NSArray *) hosts names: (NSArray *) names offset: (int64_t) offset length: (int64_t) length;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSArray *) hosts;
+- (void) setHosts: (NSArray *) hosts;
+- (BOOL) hostsIsSet;
+
+- (NSArray *) names;
+- (void) setNames: (NSArray *) names;
+- (BOOL) namesIsSet;
+
+- (int64_t) offset;
+- (void) setOffset: (int64_t) offset;
+- (BOOL) offsetIsSet;
+
+- (int64_t) length;
+- (void) setLength: (int64_t) length;
+- (BOOL) lengthIsSet;
+
+@end
+
+@interface MalformedInputException : NSException {
+ NSString * __message;
+
+ BOOL __message_isset;
+}
+
+- (id) initWithMessage: (NSString *) message;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) message;
+- (void) setMessage: (NSString *) message;
+- (BOOL) messageIsSet;
+
+@end
+
+@interface ThriftIOException : NSException {
+ NSString * __message;
+
+ BOOL __message_isset;
+}
+
+- (id) initWithMessage: (NSString *) message;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) message;
+- (void) setMessage: (NSString *) message;
+- (BOOL) messageIsSet;
+
+@end
+
+@protocol ThriftHadoopFileSystem <NSObject>
+- (void) setInactivityTimeoutPeriod: (int64_t) periodInSeconds; // throws TException
+- (void) shutdown: (int32_t) status; // throws TException
+- (ThriftHandle *) create: (Pathname *) path; // throws ThriftIOException *, TException
+- (ThriftHandle *) createFile: (Pathname *) path : (int16_t) mode : (BOOL) overwrite : (int32_t) bufferSize : (int16_t) block_replication : (int64_t) blocksize; // throws ThriftIOException *, TException
+- (ThriftHandle *) open: (Pathname *) path; // throws ThriftIOException *, TException
+- (ThriftHandle *) append: (Pathname *) path; // throws ThriftIOException *, TException
+- (BOOL) write: (ThriftHandle *) handle : (NSString *) data; // throws ThriftIOException *, TException
+- (NSString *) read: (ThriftHandle *) handle : (int64_t) offset : (int32_t) size; // throws ThriftIOException *, TException
+- (BOOL) close: (ThriftHandle *) out; // throws ThriftIOException *, TException
+- (BOOL) rm: (Pathname *) path : (BOOL) recursive; // throws ThriftIOException *, TException
+- (BOOL) rename: (Pathname *) path : (Pathname *) dest; // throws ThriftIOException *, TException
+- (BOOL) mkdirs: (Pathname *) path; // throws ThriftIOException *, TException
+- (BOOL) exists: (Pathname *) path; // throws ThriftIOException *, TException
+- (FileStatus *) stat: (Pathname *) path; // throws ThriftIOException *, TException
+- (NSArray *) listStatus: (Pathname *) path; // throws ThriftIOException *, TException
+- (void) chmod: (Pathname *) path : (int16_t) mode; // throws ThriftIOException *, TException
+- (void) chown: (Pathname *) path : (NSString *) owner : (NSString *) group; // throws ThriftIOException *, TException
+- (void) setReplication: (Pathname *) path : (int16_t) replication; // throws ThriftIOException *, TException
+- (NSArray *) getFileBlockLocations: (Pathname *) path : (int64_t) start : (int64_t) length; // throws ThriftIOException *, TException
+@end
+
+@interface ThriftHadoopFileSystemClient : NSObject <ThriftHadoopFileSystem> {
+ id <TProtocol> inProtocol;
+ id <TProtocol> outProtocol;
+}
+- (id) initWithProtocol: (id <TProtocol>) protocol;
+- (id) initWithInProtocol: (id <TProtocol>) inProtocol outProtocol: (id <TProtocol>) outProtocol;
+@end
+
+@interface hadoopfsConstants {
+}
+@end