You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2012/07/31 01:31:51 UTC
svn commit: r1367365 [1/5] - in
/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/dev-support/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/ha...
Author: todd
Date: Mon Jul 30 23:31:42 2012
New Revision: 1367365
URL: http://svn.apache.org/viewvc?rev=1367365&view=rev
Log:
Merge trunk into QJM branch
Added:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSPseudoAuthenticator.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSPseudoAuthenticator.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSUtils.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenIdentifier.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenIdentifier.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManager.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManager.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManagerException.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/DelegationTokenManagerException.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/DelegationTokenManagerService.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/service/security/DelegationTokenManagerService.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestDelegationTokenManagerService.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/TestDelegationTokenManagerService.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/KerberosTestUtils.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/KerberosTestUtils.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/krb5.conf
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/resources/krb5.conf
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/
- copied from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/tree.h
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/util/tree.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryNameNodeUpgrade.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsRetries.java
- copied unchanged from r1367364, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsRetries.java
Removed:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpKerberosAuthenticator.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpPseudoAuthenticator.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/AuthFilter.java
Modified:
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_connect.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_context_handle.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_file_handle.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chmod.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_chown.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_flush.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_getattr.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_mkdir.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_open.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_read.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_readdir.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_release.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rename.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_rmdir.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_statfs.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_truncate.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_unlink.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_utimens.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_impls_write.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_options.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/jni_helper.c
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSMkdirs.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestParallelImageWrite.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestOffsetUrlInputStream.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/unit/org/apache/hadoop/hdfs/server/namenode/TestNNLeaseRecovery.java
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1363593-1367364
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml Mon Jul 30 23:31:42 2012
@@ -25,4 +25,9 @@
<Method name="destroy" />
<Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.lib.servlet.ServerWebApp" />
+ <Field name="authority" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml Mon Jul 30 23:31:42 2012
@@ -43,6 +43,8 @@
<httpfs.tomcat.dist.dir>
${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/httpfs/tomcat
</httpfs.tomcat.dist.dir>
+ <kerberos.realm>LOCALHOST</kerberos.realm>
+ <test.exclude.kerberos.test>**/TestHttpFSWithKerberos.java</test.exclude.kerberos.test>
</properties>
<dependencies>
@@ -267,6 +269,22 @@
</excludes>
</resource>
</resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ <filtering>false</filtering>
+ <excludes>
+ <exclude>krb5.conf</exclude>
+ </excludes>
+ </testResource>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>krb5.conf</include>
+ </includes>
+ </testResource>
+ </testResources>
<plugins>
<plugin>
@@ -281,6 +299,16 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<threadCount>1</threadCount>
+ <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <java.security.krb5.conf>${project.build.directory}/test-classes/krb5.conf</java.security.krb5.conf>
+ <kerberos.realm>${kerberos.realm}</kerberos.realm>
+ </systemPropertyVariables>
+ <excludes>
+ <exclude>**/${test.exclude}.java</exclude>
+ <exclude>${test.exclude.pattern}</exclude>
+ <exclude>${test.exclude.kerberos.test}</exclude>
+ </excludes>
</configuration>
</plugin>
<plugin>
@@ -396,6 +424,36 @@
<profiles>
<profile>
+ <id>testKerberos</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <test.exclude.kerberos.test>_</test.exclude.kerberos.test>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkMode>once</forkMode>
+ <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
+ <systemPropertyVariables>
+ <java.security.krb5.conf>${project.build.directory}/test-classes/krb5.conf</java.security.krb5.conf>
+ <kerberos.realm>${kerberos.realm}</kerberos.realm>
+ <httpfs.http.hostname>localhost</httpfs.http.hostname>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/TestHttpFSWithKerberos.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
<id>docs</id>
<activation>
<activeByDefault>false</activeByDefault>
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java Mon Jul 30 23:31:42 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.fs.http.client
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.DelegationTokenRenewer;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileChecksum;
@@ -28,16 +29,18 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.json.simple.parser.ParseException;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -47,30 +50,32 @@ import java.io.FileNotFoundException;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.OutputStream;
-import java.lang.reflect.Constructor;
import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
-import java.net.URLEncoder;
+import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
/**
* HttpFSServer implementation of the FileSystemAccess FileSystem.
* <p/>
* This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
*/
-public class HttpFSFileSystem extends FileSystem {
+public class HttpFSFileSystem extends FileSystem
+ implements DelegationTokenRenewer.Renewable {
- public static final String SERVICE_NAME = "/webhdfs";
+ public static final String SERVICE_NAME = HttpFSUtils.SERVICE_NAME;
- public static final String SERVICE_VERSION = "/v1";
+ public static final String SERVICE_VERSION = HttpFSUtils.SERVICE_VERSION;
- public static final String SERVICE_PREFIX = SERVICE_NAME + SERVICE_VERSION;
+ public static final String SCHEME = "webhdfs";
public static final String OP_PARAM = "op";
public static final String DO_AS_PARAM = "doas";
@@ -84,7 +89,6 @@ public class HttpFSFileSystem extends Fi
public static final String GROUP_PARAM = "group";
public static final String MODIFICATION_TIME_PARAM = "modificationtime";
public static final String ACCESS_TIME_PARAM = "accesstime";
- public static final String RENEWER_PARAM = "renewer";
public static final Short DEFAULT_PERMISSION = 0755;
@@ -144,9 +148,6 @@ public class HttpFSFileSystem extends Fi
public static final String CONTENT_SUMMARY_SPACE_CONSUMED_JSON = "spaceConsumed";
public static final String CONTENT_SUMMARY_SPACE_QUOTA_JSON = "spaceQuota";
- public static final String DELEGATION_TOKEN_JSON = "Token";
- public static final String DELEGATION_TOKEN_URL_STRING_JSON = "urlString";
-
public static final String ERROR_JSON = "RemoteException";
public static final String ERROR_EXCEPTION_JSON = "exception";
public static final String ERROR_CLASSNAME_JSON = "javaClassName";
@@ -184,8 +185,31 @@ public class HttpFSFileSystem extends Fi
private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
private URI uri;
+ private InetSocketAddress httpFSAddr;
private Path workingDir;
+ private UserGroupInformation realUser;
private String doAs;
+ private Token<?> delegationToken;
+
+ //This method enables handling UGI doAs with SPNEGO, we have to
+ //fallback to the realuser who logged in with Kerberos credentials
+ private <T> T doAsRealUserIfNecessary(final Callable<T> callable)
+ throws IOException {
+ try {
+ if (realUser.getShortUserName().equals(doAs)) {
+ return callable.call();
+ } else {
+ return realUser.doAs(new PrivilegedExceptionAction<T>() {
+ @Override
+ public T run() throws Exception {
+ return callable.call();
+ }
+ });
+ }
+ } catch (Exception ex) {
+ throw new IOException(ex.toString(), ex);
+ }
+ }
/**
* Convenience method that creates a <code>HttpURLConnection</code> for the
@@ -204,25 +228,23 @@ public class HttpFSFileSystem extends Fi
*
* @throws IOException thrown if an IO error occurrs.
*/
- private HttpURLConnection getConnection(String method, Map<String, String> params,
- Path path, boolean makeQualified) throws IOException {
- params.put(DO_AS_PARAM, doAs);
+ private HttpURLConnection getConnection(final String method,
+ Map<String, String> params, Path path, boolean makeQualified)
+ throws IOException {
+ if (!realUser.getShortUserName().equals(doAs)) {
+ params.put(DO_AS_PARAM, doAs);
+ }
+ HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken);
if (makeQualified) {
path = makeQualified(path);
}
- URI uri = path.toUri();
- StringBuilder sb = new StringBuilder();
- sb.append(uri.getScheme()).append("://").append(uri.getAuthority()).
- append(SERVICE_PREFIX).append(uri.getPath());
-
- String separator = "?";
- for (Map.Entry<String, String> entry : params.entrySet()) {
- sb.append(separator).append(entry.getKey()).append("=").
- append(URLEncoder.encode(entry.getValue(), "UTF8"));
- separator = "&";
- }
- URL url = new URL(sb.toString());
- return getConnection(url, method);
+ final URL url = HttpFSUtils.createHttpURL(path, params);
+ return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() {
+ @Override
+ public HttpURLConnection call() throws Exception {
+ return getConnection(url, method);
+ }
+ });
}
/**
@@ -240,7 +262,8 @@ public class HttpFSFileSystem extends Fi
*/
private HttpURLConnection getConnection(URL url, String method) throws IOException {
Class<? extends Authenticator> klass =
- getConf().getClass("httpfs.authenticator.class", HttpKerberosAuthenticator.class, Authenticator.class);
+ getConf().getClass("httpfs.authenticator.class",
+ HttpFSKerberosAuthenticator.class, Authenticator.class);
Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf());
try {
HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken);
@@ -255,63 +278,6 @@ public class HttpFSFileSystem extends Fi
}
/**
- * Convenience method that JSON Parses the <code>InputStream</code> of a <code>HttpURLConnection</code>.
- *
- * @param conn the <code>HttpURLConnection</code>.
- *
- * @return the parsed JSON object.
- *
- * @throws IOException thrown if the <code>InputStream</code> could not be JSON parsed.
- */
- private static Object jsonParse(HttpURLConnection conn) throws IOException {
- try {
- JSONParser parser = new JSONParser();
- return parser.parse(new InputStreamReader(conn.getInputStream()));
- } catch (ParseException ex) {
- throw new IOException("JSON parser error, " + ex.getMessage(), ex);
- }
- }
-
- /**
- * Validates the status of an <code>HttpURLConnection</code> against an expected HTTP
- * status code. If the current status code is not the expected one it throws an exception
- * with a detail message using Server side error messages if available.
- *
- * @param conn the <code>HttpURLConnection</code>.
- * @param expected the expected HTTP status code.
- *
- * @throws IOException thrown if the current status code does not match the expected one.
- */
- private static void validateResponse(HttpURLConnection conn, int expected) throws IOException {
- int status = conn.getResponseCode();
- if (status != expected) {
- try {
- JSONObject json = (JSONObject) jsonParse(conn);
- json = (JSONObject) json.get(ERROR_JSON);
- String message = (String) json.get(ERROR_MESSAGE_JSON);
- String exception = (String) json.get(ERROR_EXCEPTION_JSON);
- String className = (String) json.get(ERROR_CLASSNAME_JSON);
-
- try {
- ClassLoader cl = HttpFSFileSystem.class.getClassLoader();
- Class klass = cl.loadClass(className);
- Constructor constr = klass.getConstructor(String.class);
- throw (IOException) constr.newInstance(message);
- } catch (IOException ex) {
- throw ex;
- } catch (Exception ex) {
- throw new IOException(MessageFormat.format("{0} - {1}", exception, message));
- }
- } catch (IOException ex) {
- if (ex.getCause() instanceof IOException) {
- throw (IOException) ex.getCause();
- }
- throw new IOException(MessageFormat.format("HTTP status [{0}], {1}", status, conn.getResponseMessage()));
- }
- }
- }
-
- /**
* Called after a new FileSystem instance is constructed.
*
* @param name a uri whose authority section names the host, port, etc. for this FileSystem
@@ -320,15 +286,28 @@ public class HttpFSFileSystem extends Fi
@Override
public void initialize(URI name, Configuration conf) throws IOException {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- doAs = ugi.getUserName();
+
+ //the real use is the one that has the Kerberos credentials needed for
+ //SPNEGO to work
+ realUser = ugi.getRealUser();
+ if (realUser == null) {
+ realUser = UserGroupInformation.getLoginUser();
+ }
+ doAs = ugi.getShortUserName();
super.initialize(name, conf);
try {
- uri = new URI(name.getScheme() + "://" + name.getHost() + ":" + name.getPort());
+ uri = new URI(name.getScheme() + "://" + name.getAuthority());
+ httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString());
} catch (URISyntaxException ex) {
throw new IOException(ex);
}
}
+ @Override
+ public String getScheme() {
+ return SCHEME;
+ }
+
/**
* Returns a URI whose scheme and authority identify this FileSystem.
*
@@ -340,6 +319,16 @@ public class HttpFSFileSystem extends Fi
}
/**
+ * Get the default port for this file system.
+ * @return the default port or 0 if there isn't one
+ */
+ @Override
+ protected int getDefaultPort() {
+ return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
+ DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
+ }
+
+ /**
* HttpFSServer subclass of the <code>FSDataInputStream</code>.
* <p/>
* This implementation does not support the
@@ -397,7 +386,7 @@ public class HttpFSFileSystem extends Fi
params.put(OP_PARAM, Operation.OPEN.toString());
HttpURLConnection conn = getConnection(Operation.OPEN.getMethod(), params,
f, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
return new FSDataInputStream(
new HttpFSDataInputStream(conn.getInputStream(), bufferSize));
}
@@ -424,7 +413,7 @@ public class HttpFSFileSystem extends Fi
try {
super.close();
} finally {
- validateResponse(conn, closeStatus);
+ HttpFSUtils.validateResponse(conn, closeStatus);
}
}
@@ -460,11 +449,11 @@ public class HttpFSFileSystem extends Fi
OutputStream os = new BufferedOutputStream(conn.getOutputStream(), bufferSize);
return new HttpFSDataOutputStream(conn, os, expectedStatus, statistics);
} catch (IOException ex) {
- validateResponse(conn, expectedStatus);
+ HttpFSUtils.validateResponse(conn, expectedStatus);
throw ex;
}
} else {
- validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
+ HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
throw new IOException("Missing HTTP 'Location' header for [" + conn.getURL() + "]");
}
} else {
@@ -476,7 +465,7 @@ public class HttpFSFileSystem extends Fi
if (exceptionAlreadyHandled) {
throw ex;
} else {
- validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
+ HttpFSUtils.validateResponse(conn, HTTP_TEMPORARY_REDIRECT);
throw ex;
}
}
@@ -548,8 +537,8 @@ public class HttpFSFileSystem extends Fi
params.put(DESTINATION_PARAM, dst.toString());
HttpURLConnection conn = getConnection(Operation.RENAME.getMethod(),
params, src, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) jsonParse(conn);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
return (Boolean) json.get(RENAME_JSON);
}
@@ -584,8 +573,8 @@ public class HttpFSFileSystem extends Fi
params.put(RECURSIVE_PARAM, Boolean.toString(recursive));
HttpURLConnection conn = getConnection(Operation.DELETE.getMethod(),
params, f, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) jsonParse(conn);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
return (Boolean) json.get(DELETE_JSON);
}
@@ -605,8 +594,8 @@ public class HttpFSFileSystem extends Fi
params.put(OP_PARAM, Operation.LISTSTATUS.toString());
HttpURLConnection conn = getConnection(Operation.LISTSTATUS.getMethod(),
params, f, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) jsonParse(conn);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
json = (JSONObject) json.get(FILE_STATUSES_JSON);
JSONArray jsonArray = (JSONArray) json.get(FILE_STATUS_JSON);
FileStatus[] array = new FileStatus[jsonArray.size()];
@@ -653,8 +642,8 @@ public class HttpFSFileSystem extends Fi
params.put(PERMISSION_PARAM, permissionToString(permission));
HttpURLConnection conn = getConnection(Operation.MKDIRS.getMethod(),
params, f, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) jsonParse(conn);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
return (Boolean) json.get(MKDIRS_JSON);
}
@@ -674,8 +663,8 @@ public class HttpFSFileSystem extends Fi
params.put(OP_PARAM, Operation.GETFILESTATUS.toString());
HttpURLConnection conn = getConnection(Operation.GETFILESTATUS.getMethod(),
params, f, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) jsonParse(conn);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
json = (JSONObject) json.get(FILE_STATUS_JSON);
f = makeQualified(f);
return createFileStatus(f, json);
@@ -693,8 +682,8 @@ public class HttpFSFileSystem extends Fi
HttpURLConnection conn =
getConnection(Operation.GETHOMEDIRECTORY.getMethod(), params,
new Path(getUri().toString(), "/"), false);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) jsonParse(conn);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
return new Path((String) json.get(HOME_DIR_JSON));
} catch (IOException ex) {
throw new RuntimeException(ex);
@@ -718,7 +707,7 @@ public class HttpFSFileSystem extends Fi
params.put(GROUP_PARAM, groupname);
HttpURLConnection conn = getConnection(Operation.SETOWNER.getMethod(),
params, p, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
}
/**
@@ -733,7 +722,7 @@ public class HttpFSFileSystem extends Fi
params.put(OP_PARAM, Operation.SETPERMISSION.toString());
params.put(PERMISSION_PARAM, permissionToString(permission));
HttpURLConnection conn = getConnection(Operation.SETPERMISSION.getMethod(), params, p, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
}
/**
@@ -755,7 +744,7 @@ public class HttpFSFileSystem extends Fi
params.put(ACCESS_TIME_PARAM, Long.toString(atime));
HttpURLConnection conn = getConnection(Operation.SETTIMES.getMethod(),
params, p, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
}
/**
@@ -777,19 +766,11 @@ public class HttpFSFileSystem extends Fi
params.put(REPLICATION_PARAM, Short.toString(replication));
HttpURLConnection conn =
getConnection(Operation.SETREPLICATION.getMethod(), params, src, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json = (JSONObject) jsonParse(conn);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
return (Boolean) json.get(SET_REPLICATION_JSON);
}
- /**
- * Creates a <code>FileStatus</code> object using a JSON file-status payload
- * received from a HttpFSServer server.
- *
- * @param json a JSON file-status payload received from a HttpFSServer server
- *
- * @return the corresponding <code>FileStatus</code>
- */
private FileStatus createFileStatus(Path parent, JSONObject json) {
String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
@@ -828,9 +809,9 @@ public class HttpFSFileSystem extends Fi
params.put(OP_PARAM, Operation.GETCONTENTSUMMARY.toString());
HttpURLConnection conn =
getConnection(Operation.GETCONTENTSUMMARY.getMethod(), params, f, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- JSONObject json =
- (JSONObject) ((JSONObject) jsonParse(conn)).get(CONTENT_SUMMARY_JSON);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ JSONObject json = (JSONObject) ((JSONObject)
+ HttpFSUtils.jsonParse(conn)).get(CONTENT_SUMMARY_JSON);
return new ContentSummary((Long) json.get(CONTENT_SUMMARY_LENGTH_JSON),
(Long) json.get(CONTENT_SUMMARY_FILE_COUNT_JSON),
(Long) json.get(CONTENT_SUMMARY_DIRECTORY_COUNT_JSON),
@@ -846,9 +827,9 @@ public class HttpFSFileSystem extends Fi
params.put(OP_PARAM, Operation.GETFILECHECKSUM.toString());
HttpURLConnection conn =
getConnection(Operation.GETFILECHECKSUM.getMethod(), params, f, true);
- validateResponse(conn, HttpURLConnection.HTTP_OK);
- final JSONObject json =
- (JSONObject) ((JSONObject) jsonParse(conn)).get(FILE_CHECKSUM_JSON);
+ HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+ final JSONObject json = (JSONObject) ((JSONObject)
+ HttpFSUtils.jsonParse(conn)).get(FILE_CHECKSUM_JSON);
return new FileChecksum() {
@Override
public String getAlgorithmName() {
@@ -877,4 +858,56 @@ public class HttpFSFileSystem extends Fi
};
}
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public Token<?> getDelegationToken(final String renewer)
+ throws IOException {
+ return doAsRealUserIfNecessary(new Callable<Token<?>>() {
+ @Override
+ public Token<?> call() throws Exception {
+ return HttpFSKerberosAuthenticator.
+ getDelegationToken(uri, httpFSAddr, authToken, renewer);
+ }
+ });
+ }
+
+
+ @Override
+ public List<Token<?>> getDelegationTokens(final String renewer)
+ throws IOException {
+ return doAsRealUserIfNecessary(new Callable<List<Token<?>>>() {
+ @Override
+ public List<Token<?>> call() throws Exception {
+ return HttpFSKerberosAuthenticator.
+ getDelegationTokens(uri, httpFSAddr, authToken, renewer);
+ }
+ });
+ }
+
+ public long renewDelegationToken(final Token<?> token) throws IOException {
+ return doAsRealUserIfNecessary(new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ return HttpFSKerberosAuthenticator.
+ renewDelegationToken(uri, authToken, token);
+ }
+ });
+ }
+
+ public void cancelDelegationToken(final Token<?> token) throws IOException {
+ HttpFSKerberosAuthenticator.
+ cancelDelegationToken(uri, authToken, token);
+ }
+
+ @Override
+ public Token<?> getRenewToken() {
+ return delegationToken;
+ }
+
+ @Override
+ public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
+ delegationToken = token;
+ }
+
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java Mon Jul 30 23:31:42 2012
@@ -70,7 +70,7 @@ public class HttpFSServerWebApp extends
/**
* Constructor used for testing purposes.
*/
- protected HttpFSServerWebApp(String homeDir, String configDir, String logDir,
+ public HttpFSServerWebApp(String homeDir, String configDir, String logDir,
String tempDir, Configuration config) {
super(NAME, homeDir, configDir, logDir, tempDir, config);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/ServerException.java Mon Jul 30 23:31:42 2012
@@ -39,7 +39,11 @@ public class ServerException extends XEx
S08("Could not load service classes, {0}"),
S09("Could not set service [{0}] programmatically -server shutting down-, {1}"),
S10("Service [{0}] requires service [{1}]"),
- S11("Service [{0}] exception during status change to [{1}] -server shutting down-, {2}");
+ S11("Service [{0}] exception during status change to [{1}] -server shutting down-, {2}"),
+ S12("Could not start service [{0}], {1}"),
+ S13("Missing system property [{0}]"),
+ S14("Could not initialize server, {0}")
+ ;
private String msg;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java Mon Jul 30 23:31:42 2012
@@ -18,12 +18,16 @@
package org.apache.hadoop.lib.servlet;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.lib.server.Server;
import org.apache.hadoop.lib.server.ServerException;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import java.text.MessageFormat;
/**
@@ -36,9 +40,13 @@ public abstract class ServerWebApp exten
private static final String CONFIG_DIR = ".config.dir";
private static final String LOG_DIR = ".log.dir";
private static final String TEMP_DIR = ".temp.dir";
+ private static final String HTTP_HOSTNAME = ".http.hostname";
+ private static final String HTTP_PORT = ".http.port";
private static ThreadLocal<String> HOME_DIR_TL = new ThreadLocal<String>();
+ private InetSocketAddress authority;
+
/**
* Method for testing purposes.
*/
@@ -147,6 +155,38 @@ public abstract class ServerWebApp exten
}
/**
+ * Resolves the host & port InetSocketAddress the web server is listening to.
+ * <p/>
+ * This implementation looks for the following 2 properties:
+ * <ul>
+ * <li>#SERVER_NAME#.http.hostname</li>
+ * <li>#SERVER_NAME#.http.port</li>
+ * </ul>
+ *
+ * @return the host & port InetSocketAddress the web server is listening to.
+ * @throws ServerException thrown if any of the above 2 properties is not defined.
+ */
+ protected InetSocketAddress resolveAuthority() throws ServerException {
+ String hostnameKey = getName() + HTTP_HOSTNAME;
+ String portKey = getName() + HTTP_PORT;
+ String host = System.getProperty(hostnameKey);
+ String port = System.getProperty(portKey);
+ if (host == null) {
+ throw new ServerException(ServerException.ERROR.S13, hostnameKey);
+ }
+ if (port == null) {
+ throw new ServerException(ServerException.ERROR.S13, portKey);
+ }
+ try {
+ InetAddress add = InetAddress.getByName(hostnameKey);
+ int portNum = Integer.parseInt(port);
+ return new InetSocketAddress(add, portNum);
+ } catch (UnknownHostException ex) {
+ throw new ServerException(ServerException.ERROR.S14, ex.toString(), ex);
+ }
+ }
+
+ /**
* Destroys the <code>ServletContextListener</code> which destroys
* the Server.
*
@@ -156,4 +196,29 @@ public abstract class ServerWebApp exten
destroy();
}
+ /**
+ * Returns the hostname:port InetSocketAddress the webserver is listening to.
+ *
+ * @return the hostname:port InetSocketAddress the webserver is listening to.
+ */
+ public InetSocketAddress getAuthority() throws ServerException {
+ synchronized (this) {
+ if (authority == null) {
+ authority = resolveAuthority();
+ }
+ }
+ return authority;
+ }
+
+ /**
+ * Sets an alternate hostname:port InetSocketAddress to use.
+ * <p/>
+ * For testing purposes.
+ *
+ * @param authority alterante authority.
+ */
+ @VisibleForTesting
+ public void setAuthority(InetSocketAddress authority) {
+ this.authority = authority;
+ }
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml Mon Jul 30 23:31:42 2012
@@ -35,6 +35,7 @@
org.apache.hadoop.lib.service.scheduler.SchedulerService,
org.apache.hadoop.lib.service.security.GroupsService,
org.apache.hadoop.lib.service.security.ProxyUserService,
+ org.apache.hadoop.lib.service.security.DelegationTokenManagerService,
org.apache.hadoop.lib.service.hadoop.FileSystemAccessService
</value>
<description>
@@ -88,12 +89,12 @@
<description>
Defines the authentication mechanism used by httpfs for its HTTP clients.
- Valid values are 'simple' and 'kerberos'.
+ Valid values are 'simple' or 'kerberos'.
If using 'simple' HTTP clients must specify the username with the
'user.name' query string parameter.
- If using 'kerberos' HTTP clients must use HTTP SPNEGO.
+ If using 'kerberos' HTTP clients must use HTTP SPNEGO or delegation tokens.
</description>
</property>
@@ -153,6 +154,32 @@
</description>
</property>
+ <!-- HttpFS Delegation Token configuration -->
+
+ <property>
+ <name>httpfs.delegation.token.manager.update.interval</name>
+ <value>86400</value>
+ <description>
+ HttpFS delegation token update interval, default 1 day, in seconds.
+ </description>
+ </property>
+
+ <property>
+ <name>httpfs.delegation.token.manager.max.lifetime</name>
+ <value>604800</value>
+ <description>
+ HttpFS delegation token maximum lifetime, default 7 days, in seconds
+ </description>
+ </property>
+
+ <property>
+ <name>httpfs.delegation.token.manager.renewal.interval</name>
+ <value>86400</value>
+ <description>
+ HttpFS delegation token update interval, default 1 day, in seconds.
+ </description>
+ </property>
+
<!-- FileSystemAccess Namenode Security Configuration -->
<property>
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/webapp/WEB-INF/web.xml Mon Jul 30 23:31:42 2012
@@ -47,7 +47,7 @@
<filter>
<filter-name>authFilter</filter-name>
- <filter-class>org.apache.hadoop.fs.http.server.AuthFilter</filter-class>
+ <filter-class>org.apache.hadoop.fs.http.server.HttpFSAuthenticationFilter</filter-class>
</filter>
<filter>
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystem.java Mon Jul 30 23:31:42 2012
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Writer;
+import java.net.URI;
import java.net.URL;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@@ -100,16 +101,24 @@ public class TestHttpFSFileSystem extend
server.start();
}
+ protected Class getFileSystemClass() {
+ return HttpFSFileSystem.class;
+ }
+
protected FileSystem getHttpFileSystem() throws Exception {
Configuration conf = new Configuration();
- conf.set("fs.http.impl", HttpFSFileSystem.class.getName());
- return FileSystem.get(TestJettyHelper.getJettyURL().toURI(), conf);
+ conf.set("fs.webhdfs.impl", getFileSystemClass().getName());
+ URI uri = new URI("webhdfs://" +
+ TestJettyHelper.getJettyURL().toURI().getAuthority());
+ return FileSystem.get(uri, conf);
}
protected void testGet() throws Exception {
FileSystem fs = getHttpFileSystem();
Assert.assertNotNull(fs);
- Assert.assertEquals(fs.getUri(), TestJettyHelper.getJettyURL().toURI());
+ URI uri = new URI("webhdfs://" +
+ TestJettyHelper.getJettyURL().toURI().getAuthority());
+ Assert.assertEquals(fs.getUri(), uri);
fs.close();
}
@@ -474,8 +483,9 @@ public class TestHttpFSFileSystem extend
for (int i = 0; i < Operation.values().length; i++) {
ops[i] = new Object[]{Operation.values()[i]};
}
+ //To test one or a subset of operations do:
+ //return Arrays.asList(new Object[][]{ new Object[]{Operation.OPEN}});
return Arrays.asList(ops);
-// return Arrays.asList(new Object[][]{ new Object[]{Operation.CREATE}});
}
private Operation operation;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestWebhdfsFileSystem.java Mon Jul 30 23:31:42 2012
@@ -36,20 +36,8 @@ public class TestWebhdfsFileSystem exten
}
@Override
- protected FileSystem getHttpFileSystem() throws Exception {
- Configuration conf = new Configuration();
- conf.set("fs.webhdfs.impl", WebHdfsFileSystem.class.getName());
- URI uri = new URI("webhdfs://" + TestJettyHelper.getJettyURL().toURI().getAuthority());
- return FileSystem.get(uri, conf);
- }
-
- @Override
- protected void testGet() throws Exception {
- FileSystem fs = getHttpFileSystem();
- Assert.assertNotNull(fs);
- URI uri = new URI("webhdfs://" + TestJettyHelper.getJettyURL().toURI().getAuthority());
- Assert.assertEquals(fs.getUri(), uri);
- fs.close();
+ protected Class getFileSystemClass() {
+ return WebHdfsFileSystem.class;
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java Mon Jul 30 23:31:42 2012
@@ -15,11 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.fs.http.server;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import org.junit.Assert;
import java.io.BufferedReader;
import java.io.File;
@@ -39,9 +37,13 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator;
import org.apache.hadoop.lib.server.Service;
import org.apache.hadoop.lib.server.ServiceException;
import org.apache.hadoop.lib.service.Groups;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.server.AuthenticationToken;
+import org.apache.hadoop.security.authentication.util.Signer;
import org.apache.hadoop.test.HFSTestCase;
import org.apache.hadoop.test.HadoopUsersConfTestHelper;
import org.apache.hadoop.test.TestDir;
@@ -50,6 +52,8 @@ import org.apache.hadoop.test.TestHdfs;
import org.apache.hadoop.test.TestHdfsHelper;
import org.apache.hadoop.test.TestJetty;
import org.apache.hadoop.test.TestJettyHelper;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
import org.junit.Test;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
@@ -103,11 +107,13 @@ public class TestHttpFSServer extends HF
}
}
- private void createHttpFSServer() throws Exception {
+
+ private void createHttpFSServer(boolean addDelegationTokenAuthHandler)
+ throws Exception {
File homeDir = TestDirHelper.getTestDir();
- assertTrue(new File(homeDir, "conf").mkdir());
- assertTrue(new File(homeDir, "log").mkdir());
- assertTrue(new File(homeDir, "temp").mkdir());
+ Assert.assertTrue(new File(homeDir, "conf").mkdir());
+ Assert.assertTrue(new File(homeDir, "log").mkdir());
+ Assert.assertTrue(new File(homeDir, "temp").mkdir());
HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath());
File secretFile = new File(new File(homeDir, "conf"), "secret");
@@ -128,6 +134,10 @@ public class TestHttpFSServer extends HF
//HTTPFS configuration
conf = new Configuration(false);
+ if (addDelegationTokenAuthHandler) {
+ conf.set("httpfs.authentication.type",
+ HttpFSKerberosAuthenticationHandlerForTesting.class.getName());
+ }
conf.set("httpfs.services.ext", MockGroups.class.getName());
conf.set("httpfs.admin.group", HadoopUsersConfTestHelper.
getHadoopUserGroups(HadoopUsersConfTestHelper.getHadoopUsers()[0])[0]);
@@ -147,6 +157,9 @@ public class TestHttpFSServer extends HF
Server server = TestJettyHelper.getJettyServer();
server.addHandler(context);
server.start();
+ if (addDelegationTokenAuthHandler) {
+ HttpFSServerWebApp.get().setAuthority(TestJettyHelper.getAuthority());
+ }
}
@Test
@@ -154,28 +167,28 @@ public class TestHttpFSServer extends HF
@TestJetty
@TestHdfs
public void instrumentation() throws Exception {
- createHttpFSServer();
+ createHttpFSServer(false);
URL url = new URL(TestJettyHelper.getJettyURL(),
MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation", "nobody"));
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_UNAUTHORIZED);
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_UNAUTHORIZED);
url = new URL(TestJettyHelper.getJettyURL(),
MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation",
HadoopUsersConfTestHelper.getHadoopUsers()[0]));
conn = (HttpURLConnection) url.openConnection();
- assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String line = reader.readLine();
reader.close();
- assertTrue(line.contains("\"counters\":{"));
+ Assert.assertTrue(line.contains("\"counters\":{"));
url = new URL(TestJettyHelper.getJettyURL(),
MessageFormat.format("/webhdfs/v1/foo?user.name={0}&op=instrumentation",
HadoopUsersConfTestHelper.getHadoopUsers()[0]));
conn = (HttpURLConnection) url.openConnection();
- assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST);
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST);
}
@Test
@@ -183,13 +196,13 @@ public class TestHttpFSServer extends HF
@TestJetty
@TestHdfs
public void testHdfsAccess() throws Exception {
- createHttpFSServer();
+ createHttpFSServer(false);
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
URL url = new URL(TestJettyHelper.getJettyURL(),
MessageFormat.format("/webhdfs/v1/?user.name={0}&op=liststatus", user));
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
reader.readLine();
reader.close();
@@ -200,7 +213,7 @@ public class TestHttpFSServer extends HF
@TestJetty
@TestHdfs
public void testGlobFilter() throws Exception {
- createHttpFSServer();
+ createHttpFSServer(false);
FileSystem fs = FileSystem.get(TestHdfsHelper.getHdfsConf());
fs.mkdirs(new Path("/tmp"));
@@ -210,7 +223,7 @@ public class TestHttpFSServer extends HF
URL url = new URL(TestJettyHelper.getJettyURL(),
MessageFormat.format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*", user));
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK);
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
reader.readLine();
reader.close();
@@ -221,7 +234,7 @@ public class TestHttpFSServer extends HF
@TestJetty
@TestHdfs
public void testPutNoOperation() throws Exception {
- createHttpFSServer();
+ createHttpFSServer(false);
String user = HadoopUsersConfTestHelper.getHadoopUsers()[0];
URL url = new URL(TestJettyHelper.getJettyURL(),
@@ -230,7 +243,87 @@ public class TestHttpFSServer extends HF
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setRequestMethod("PUT");
- assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST);
+ Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_BAD_REQUEST);
+ }
+
+ @Test
+ @TestDir
+ @TestJetty
+ @TestHdfs
+ public void testDelegationTokenOperations() throws Exception {
+ createHttpFSServer(true);
+
+ URL url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETHOMEDIRECTORY");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
+ conn.getResponseCode());
+
+
+ AuthenticationToken token =
+ new AuthenticationToken("u", "p",
+ HttpFSKerberosAuthenticationHandlerForTesting.TYPE);
+ token.setExpires(System.currentTimeMillis() + 100000000);
+ Signer signer = new Signer("secret".getBytes());
+ String tokenSigned = signer.sign(token.toString());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETHOMEDIRECTORY");
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestProperty("Cookie",
+ AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK,
+ conn.getResponseCode());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETDELEGATIONTOKEN");
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestProperty("Cookie",
+ AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK,
+ conn.getResponseCode());
+
+ JSONObject json = (JSONObject)
+ new JSONParser().parse(new InputStreamReader(conn.getInputStream()));
+ json = (JSONObject)
+ json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON);
+ String tokenStr = (String)
+ json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON);
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
+ conn = (HttpURLConnection) url.openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_OK,
+ conn.getResponseCode());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod("PUT");
+ Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
+ conn.getResponseCode());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod("PUT");
+ conn.setRequestProperty("Cookie",
+ AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK,
+ conn.getResponseCode());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr);
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod("PUT");
+ Assert.assertEquals(HttpURLConnection.HTTP_OK,
+ conn.getResponseCode());
+
+ url = new URL(TestJettyHelper.getJettyURL(),
+ "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
+ conn = (HttpURLConnection) url.openConnection();
+ Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,
+ conn.getResponseCode());
}
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestDirHelper.java Mon Jul 30 23:31:42 2012
@@ -73,7 +73,7 @@ public class TestDirHelper implements Me
System.exit(-1);
}
- TEST_DIR_ROOT = new File(TEST_DIR_ROOT, "testdir").getAbsolutePath();
+ TEST_DIR_ROOT = new File(TEST_DIR_ROOT, "test-dir").getAbsolutePath();
System.setProperty(TEST_DIR_PROP, TEST_DIR_ROOT);
File dir = new File(TEST_DIR_ROOT);
@@ -83,8 +83,6 @@ public class TestDirHelper implements Me
System.exit(-1);
}
- System.setProperty("test.circus", "true");
-
System.out.println(">>> " + TEST_DIR_PROP + " : " + System.getProperty(TEST_DIR_PROP));
} catch (IOException ex) {
throw new RuntimeException(ex);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java Mon Jul 30 23:31:42 2012
@@ -18,9 +18,11 @@
package org.apache.hadoop.test;
import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.ServerSocket;
import java.net.URL;
+import java.net.UnknownHostException;
import org.junit.Test;
import org.junit.rules.MethodRule;
@@ -65,9 +67,9 @@ public class TestJettyHelper implements
private Server createJettyServer() {
try {
-
- String host = InetAddress.getLocalHost().getHostName();
- ServerSocket ss = new ServerSocket(0);
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket(0, 50, localhost);
int port = ss.getLocalPort();
ss.close();
Server server = new Server(0);
@@ -80,6 +82,23 @@ public class TestJettyHelper implements
}
/**
+ * Returns the authority (hostname & port) used by the JettyServer.
+ *
+ * @return an <code>InetSocketAddress</code> with the corresponding authority.
+ */
+ public static InetSocketAddress getAuthority() {
+ Server server = getJettyServer();
+ try {
+ InetAddress add =
+ InetAddress.getByName(server.getConnectors()[0].getHost());
+ int port = server.getConnectors()[0].getPort();
+ return new InetSocketAddress(add, port);
+ } catch (UnknownHostException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
* Returns a Jetty server ready to be configured and the started. This server
* is only available when the test method has been annotated with
* {@link TestJetty}. Refer to {@link HTestCase} header for details.
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Jul 30 23:31:42 2012
@@ -109,8 +109,6 @@ Trunk (unreleased changes)
HDFS-3630 Modify TestPersistBlocks to use both flush and hflush (sanjay)
- HDFS-3583. Convert remaining tests to Junit4. (Andrew Wang via atm)
-
OPTIMIZATIONS
BUG FIXES
@@ -201,6 +199,8 @@ Branch-2 ( Unreleased changes )
HDFS-3518. Add a utility method HdfsUtils.isHealthy(uri) for checking if
the given HDFS is healthy. (szetszwo)
+ HDFS-3113. httpfs does not support delegation tokens. (tucu)
+
IMPROVEMENTS
HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG
@@ -305,7 +305,8 @@ Branch-2 ( Unreleased changes )
HDFS-3613. GSet prints some INFO level values, which aren't
really very useful to all (Andrew Wang via harsh)
- HDFS-3611. NameNode prints unnecessary WARNs about edit log normally skipping a few bytes. (Colin Patrick McCabe via harsh)
+ HDFS-3611. NameNode prints unnecessary WARNs about edit log normally skipping
+ a few bytes. (Colin Patrick McCabe via harsh)
HDFS-3582. Hook daemon process exit for testing. (eli)
@@ -352,6 +353,13 @@ Branch-2 ( Unreleased changes )
HDFS-1249. With fuse-dfs, chown which only has owner (or only group)
argument fails with Input/output error. (Colin Patrick McCabe via eli)
+ HDFS-3583. Convert remaining tests to Junit4. (Andrew Wang via atm)
+
+ HDFS-3711. Manually convert remaining tests to JUnit4. (Andrew Wang via atm)
+
+ HDFS-3650. Use MutableQuantiles to provide latency histograms for various
+ operations. (Andrew Wang via atm)
+
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log
@@ -363,6 +371,10 @@ Branch-2 ( Unreleased changes )
HDFS-3110. Use directRead API to reduce the number of buffer copies in
libhdfs (Henry Robinson via todd)
+ HDFS-3697. Enable fadvise readahead by default. (todd)
+
+ HDFS-3667. Add retry support to WebHdfsFileSystem. (szetszwo)
+
BUG FIXES
HDFS-3385. The last block of INodeFileUnderConstruction is not
@@ -518,6 +530,22 @@ Branch-2 ( Unreleased changes )
HDFS-3690. BlockPlacementPolicyDefault incorrectly casts LOG. (eli)
+ HDFS-3597. SNN fails to start after DFS upgrade. (Andy Isaacson via todd)
+
+ HDFS-3608. fuse_dfs: detect changes in UID ticket cache. (Colin Patrick
+ McCabe via atm)
+
+ HDFS-3709. TestStartup tests still binding to the ephemeral port. (eli)
+
+ HDFS-3720. hdfs.h must get packaged. (Colin Patrick McCabe via atm)
+
+ HDFS-3626. Creating file with invalid path can corrupt edit log (todd)
+
+ HDFS-3679. fuse_dfs notrash option sets usetrash. (Conrad Meyer via suresh)
+
+ HDFS-3732. fuse_dfs: incorrect configuration value checked for connection
+ expiry timer period. (Colin Patrick McCabe via atm)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
@@ -1388,6 +1416,11 @@ Release 0.23.3 - UNRELEASED
HDFS-3646. LeaseRenewer can hold reference to inactive DFSClient
instances forever. (Kihwal Lee via daryn)
+ HDFS-3696. Set chunked streaming mode in WebHdfsFileSystem write operations
+ to get around a Java library bug causing OutOfMemoryError. (szetszwo)
+
+ HDFS-3553. Hftp proxy tokens are broken (daryn)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt Mon Jul 30 23:31:42 2012
@@ -87,6 +87,7 @@ include_directories(
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
+ main/native
main/native/libhdfs
)
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/faultinject_framework.xml Mon Jul 30 23:31:42 2012
@@ -332,13 +332,12 @@ package org.apache.hadoop.fs;
import org.junit.Test;
import org.junit.Before;
-import junit.framework.TestCase;
-public class DemoFiTest extends TestCase {
+public class DemoFiTest {
public static final String BLOCK_RECEIVER_FAULT="hdfs.datanode.BlockReceiver";
@Override
@Before
- public void setUp(){
+ public void setUp() {
//Setting up the test's environment as required
}
Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1363593-1367364
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java Mon Jul 30 23:31:42 2012
@@ -57,9 +57,9 @@ public abstract class ByteRangeInputStre
return url;
}
- protected abstract HttpURLConnection openConnection() throws IOException;
-
- protected abstract HttpURLConnection openConnection(final long offset) throws IOException;
+ /** Connect to server with a data offset. */
+ protected abstract HttpURLConnection connect(final long offset,
+ final boolean resolved) throws IOException;
}
enum StreamStatus {
@@ -85,9 +85,6 @@ public abstract class ByteRangeInputStre
this.resolvedURL = r;
}
- protected abstract void checkResponseCode(final HttpURLConnection connection
- ) throws IOException;
-
protected abstract URL getResolvedUrl(final HttpURLConnection connection
) throws IOException;
@@ -113,13 +110,10 @@ public abstract class ByteRangeInputStre
protected InputStream openInputStream() throws IOException {
// Use the original url if no resolved url exists, eg. if
// it's the first time a request is made.
- final URLOpener opener =
- (resolvedURL.getURL() == null) ? originalURL : resolvedURL;
-
- final HttpURLConnection connection = opener.openConnection(startPos);
- connection.connect();
- checkResponseCode(connection);
+ final boolean resolved = resolvedURL.getURL() != null;
+ final URLOpener opener = resolved? resolvedURL: originalURL;
+ final HttpURLConnection connection = opener.connect(startPos, resolved);
final String cl = connection.getHeaderField(StreamFile.CONTENT_LENGTH);
if (cl == null) {
throw new IOException(StreamFile.CONTENT_LENGTH+" header is missing");
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon Jul 30 23:31:42 2012
@@ -74,7 +74,7 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY = "dfs.datanode.balance.bandwidthPerSec";
public static final long DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_DEFAULT = 1024*1024;
public static final String DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes";
- public static final long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 0;
+ public static final long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 4 * 1024 * 1024; // 4MB
public static final String DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY = "dfs.datanode.drop.cache.behind.writes";
public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_DEFAULT = false;
public static final String DFS_DATANODE_SYNC_BEHIND_WRITES_KEY = "dfs.datanode.sync.behind.writes";
@@ -203,6 +203,7 @@ public class DFSConfigKeys extends Commo
public static final String DFS_CLIENT_READ_PREFETCH_SIZE_KEY = "dfs.client.read.prefetch.size";
public static final String DFS_CLIENT_RETRY_WINDOW_BASE= "dfs.client.retry.window.base";
public static final String DFS_METRICS_SESSION_ID_KEY = "dfs.metrics.session-id";
+ public static final String DFS_METRICS_PERCENTILES_INTERVALS_KEY = "dfs.metrics.percentiles.intervals";
public static final String DFS_DATANODE_HOST_NAME_KEY = "dfs.datanode.hostname";
public static final String DFS_NAMENODE_HOSTS_KEY = "dfs.namenode.hosts";
public static final String DFS_NAMENODE_HOSTS_EXCLUDE_KEY = "dfs.namenode.hosts.exclude";
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java Mon Jul 30 23:31:42 2012
@@ -76,6 +76,8 @@ import org.apache.hadoop.util.DataChecks
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Time;
+import com.google.common.annotations.VisibleForTesting;
+
/****************************************************************
* DFSOutputStream creates files from a stream of bytes.
@@ -1210,7 +1212,8 @@ public class DFSOutputStream extends FSO
//
// returns the list of targets, if any, that is being currently used.
//
- synchronized DatanodeInfo[] getPipeline() {
+ @VisibleForTesting
+ public synchronized DatanodeInfo[] getPipeline() {
if (streamer == null) {
return null;
}
@@ -1752,11 +1755,13 @@ public class DFSOutputStream extends FSO
}
}
- void setArtificialSlowdown(long period) {
+ @VisibleForTesting
+ public void setArtificialSlowdown(long period) {
artificialSlowdown = period;
}
- synchronized void setChunksPerPacket(int value) {
+ @VisibleForTesting
+ public synchronized void setChunksPerPacket(int value) {
chunksPerPacket = Math.min(chunksPerPacket, value);
packetSize = PacketHeader.PKT_HEADER_LEN +
(checksum.getBytesPerChecksum() +
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1367365&r1=1367364&r2=1367365&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Mon Jul 30 23:31:42 2012
@@ -56,6 +56,7 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -118,7 +119,7 @@ public class DFSUtil {
/**
* Whether the pathname is valid. Currently prohibits relative paths,
- * and names which contain a ":" or "/"
+ * names which contain a ":" or "//", or other non-canonical paths.
*/
public static boolean isValidName(String src) {
// Path must be absolute.
@@ -127,15 +128,22 @@ public class DFSUtil {
}
// Check for ".." "." ":" "/"
- StringTokenizer tokens = new StringTokenizer(src, Path.SEPARATOR);
- while(tokens.hasMoreTokens()) {
- String element = tokens.nextToken();
+ String[] components = StringUtils.split(src, '/');
+ for (int i = 0; i < components.length; i++) {
+ String element = components[i];
if (element.equals("..") ||
element.equals(".") ||
(element.indexOf(":") >= 0) ||
(element.indexOf("/") >= 0)) {
return false;
}
+
+ // The string may start or end with a /, but not have
+ // "//" in the middle.
+ if (element.isEmpty() && i != components.length - 1 &&
+ i != 0) {
+ return false;
+ }
}
return true;
}