You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2013/01/22 20:33:11 UTC
svn commit: r1437113 [1/2] - in
/hadoop/common/branches/branch-trunk-win/hadoop-common-project: ./
hadoop-common/ hadoop-common/dev-support/ hadoop-common/src/
hadoop-common/src/main/bin/ hadoop-common/src/main/conf/
hadoop-common/src/main/docs/ hadoop...
Author: suresh
Date: Tue Jan 22 19:33:02 2013
New Revision: 1437113
URL: http://svn.apache.org/viewvc?rev=1437113&view=rev
Log:
Merging trunk to branch-trunk-win
Added:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
- copied unchanged from r1437106, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java
- copied unchanged from r1437106, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
- copied unchanged from r1437106, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
- copied unchanged from r1437106, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/config.h.cmake
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/service_level_auth.xml
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
hadoop/common/branches/branch-trunk-win/hadoop-common-project/pom.xml
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jan 22 19:33:02 2013
@@ -146,8 +146,13 @@ Trunk (Unreleased)
HADOOP-9162. Add utility to check native library availability.
(Binglin Chang via suresh)
+ HADOOP-8924. Add maven plugin alternative to shell script to save
+ package-info.java. (Chris Nauroth via suresh)
+
BUG FIXES
+ HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
+
HADOOP-9041. FsUrlStreamHandlerFactory could cause an infinite loop in
FileSystem initialization. (Yanbo Liang and Radim Kolar via llu)
@@ -306,9 +311,14 @@ Trunk (Unreleased)
HADOOP-9131. Turn off TestLocalFileSystem#testListStatusWithColons on
Windows. (Chris Nauroth via suresh)
- HADOOP-8957 AbstractFileSystem#IsValidName should be overridden for
+ HADOOP-8957. AbstractFileSystem#IsValidName should be overridden for
embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia)
+ HADOOP-9139. improve killKdc.sh (Ivan A. Veselovsky via bobby)
+
+ HADOOP-9202. test-patch.sh fails during mvn eclipse:eclipse if patch adds
+ a new module to the build (Chris Nauroth via bobby)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -319,6 +329,8 @@ Release 2.0.3-alpha - Unreleased
INCOMPATIBLE CHANGES
+ HADOOP-8999. SASL negotiation is flawed (daryn)
+
NEW FEATURES
HADOOP-8597. Permit FsShell's text command to read Avro files.
@@ -420,6 +432,24 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8427. Convert Forrest docs to APT, incremental. (adi2 via tucu)
+ HADOOP-9173. Add security token protobuf definition to common and
+ use it in hdfs. (suresh)
+
+ HADOOP-9119. Add test to FileSystemContractBaseTest to verify integrity
+ of overwritten files. (Steve Loughran via suresh)
+
+ HADOOP-9192. Move token related request/response messages to common.
+ (suresh)
+
+ HADOOP-8712. Change default hadoop.security.group.mapping to
+ JniBasedUnixGroupsNetgroupMappingWithFallback (Robert Parker via todd)
+
+ HADOOP-9106. Allow configuration of IPC connect timeout.
+ (Rober Parker via suresh)
+
+ HADOOP-9216. CompressionCodecFactory#getCodecClasses should trim the
+ result of parsing by Configuration. (Tsuyoshi Ozawa via todd)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
@@ -480,8 +510,6 @@ Release 2.0.3-alpha - Unreleased
HADOOP-7115. Add a cache for getpwuid_r and getpwgid_r calls (tucu)
- HADOOP-8999. SASL negotiation is flawed (daryn)
-
HADOOP-6607. Add different variants of non caching HTTP headers. (tucu)
HADOOP-9049. DelegationTokenRenewer needs to be Singleton and FileSystems
@@ -516,6 +544,31 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9153. Support createNonRecursive in ViewFileSystem.
(Sandy Ryza via tomwhite)
+ HADOOP-9181. Set daemon flag for HttpServer's QueuedThreadPool.
+ (Liang Xie via suresh)
+
+ HADOOP-9155. FsPermission should have different default value, 777 for
+ directory and 666 for file. (Binglin Chang via atm)
+
+ HADOOP-9183. Potential deadlock in ActiveStandbyElector. (tomwhite)
+
+ HADOOP-9203. RPCCallBenchmark should find a random available port.
+ (Andrew Purtell via suresh)
+
+ HADOOP-9178. src/main/conf is missing hadoop-policy.xml.
+ (Sandy Ryza via eli)
+
+ HADOOP-8816. HTTP Error 413 full HEAD if using kerberos authentication.
+ (moritzmoeller via tucu)
+
+ HADOOP-9212. Potential deadlock in FileSystem.Cache/IPC/UGI. (tomwhite)
+
+ HADOOP-9193. hadoop script can inadvertently expand wildcard arguments
+ when delegating to hdfs script. (Andy Isaacson via todd)
+
+ HADOOP-9215. when using cmake-2.6, libhadoop.so doesn't get created
+ (only libhadoop.so.1.0.0) (Colin Patrick McCabe via todd)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -1206,6 +1259,21 @@ Release 2.0.0-alpha - 05-23-2012
HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via
bobby)
+Release 0.23.7 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ HADOOP-8849. FileUtil#fullyDelete should grant the target directories +rwx
+ permissions (Ivan A. Veselovsky via bobby)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1213,6 +1281,8 @@ Release 0.23.6 - UNRELEASED
NEW FEATURES
IMPROVEMENTS
+ HADOOP-9217. Print thread dumps when hadoop-common tests fail.
+ (Andrey Klochkov via suresh)
OPTIMIZATIONS
@@ -1229,7 +1299,10 @@ Release 0.23.6 - UNRELEASED
HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby)
-Release 0.23.5 - UNRELEASED
+ HADOOP-9097. Maven RAT plugin is not checking all source files (tgraves)
+
+Release 0.23.5 - 2012-11-28
+
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1428602-1437112
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml Tue Jan 22 19:33:02 2013
@@ -282,6 +282,10 @@
<!-- protobuf generated code -->
<Class name="~org\.apache\.hadoop\.ha\.proto\.ZKFCProtocolProtos.*"/>
</Match>
+ <Match>
+ <!-- protobuf generated code -->
+ <Class name="~org\.apache\.hadoop\.security\.proto\.SecurityProtos.*"/>
+ </Match>
<!--
Manually checked, misses child thread manually syncing on parent's intrinsic lock.
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml Tue Jan 22 19:33:02 2013
@@ -431,6 +431,7 @@
<argument>src/main/proto/RpcHeader.proto</argument>
<argument>src/main/proto/ZKFCProtocol.proto</argument>
<argument>src/main/proto/ProtobufRpcEngine.proto</argument>
+ <argument>src/main/proto/Security.proto</argument>
</arguments>
</configuration>
</execution>
@@ -494,13 +495,26 @@
<exclude>dev-support/jdiff/**</exclude>
<exclude>src/main/native/*</exclude>
<exclude>src/main/native/config/*</exclude>
- <exclude>src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo</exclude>
<exclude>src/main/native/m4/*</exclude>
<exclude>src/test/empty-file</exclude>
<exclude>src/test/all-tests</exclude>
+ <exclude>src/test/resources/kdc/ldif/users.ldif</exclude>
+ <exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c</exclude>
</excludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.hadoop.test.TimedOutTestsListener</value>
+ </property>
+ </properties>
+ </configuration>
+ </plugin>
</plugins>
</build>
@@ -584,6 +598,9 @@
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
</exec>
+ <!-- The second make is a workaround for HADOOP-9215. It can
+ be removed when version 2.6 of cmake is no longer supported . -->
+ <exec executable="make" dir="${project.build.directory}/native" failonerror="true"></exec>
</target>
</configuration>
</execution>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/config.h.cmake
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/config.h.cmake?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/config.h.cmake (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/config.h.cmake Tue Jan 22 19:33:02 2013
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
#ifndef CONFIG_H
#define CONFIG_H
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop Tue Jan 22 19:33:02 2013
@@ -58,9 +58,9 @@ case $COMMAND in
#try to locate hdfs and if present, delegate to it.
shift
if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then
- exec "${HADOOP_HDFS_HOME}"/bin/hdfs ${COMMAND/dfsgroups/groups} $*
+ exec "${HADOOP_HDFS_HOME}"/bin/hdfs ${COMMAND/dfsgroups/groups} "$@"
elif [ -f "${HADOOP_PREFIX}"/bin/hdfs ]; then
- exec "${HADOOP_PREFIX}"/bin/hdfs ${COMMAND/dfsgroups/groups} $*
+ exec "${HADOOP_PREFIX}"/bin/hdfs ${COMMAND/dfsgroups/groups} "$@"
else
echo "HADOOP_HDFS_HOME not found!"
exit 1
@@ -75,9 +75,9 @@ case $COMMAND in
#try to locate mapred and if present, delegate to it.
shift
if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then
- exec "${HADOOP_MAPRED_HOME}"/bin/mapred ${COMMAND/mrgroups/groups} $*
+ exec "${HADOOP_MAPRED_HOME}"/bin/mapred ${COMMAND/mrgroups/groups} "$@"
elif [ -f "${HADOOP_PREFIX}"/bin/mapred ]; then
- exec "${HADOOP_PREFIX}"/bin/mapred ${COMMAND/mrgroups/groups} $*
+ exec "${HADOOP_PREFIX}"/bin/mapred ${COMMAND/mrgroups/groups} "$@"
else
echo "HADOOP_MAPRED_HOME not found!"
exit 1
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1428602-1437112
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/service_level_auth.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/service_level_auth.xml?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/service_level_auth.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/service_level_auth.xml Tue Jan 22 19:33:02 2013
@@ -117,22 +117,6 @@
namenode to communicate with the namenode.</td>
</tr>
<tr>
- <td><code>security.inter.tracker.protocol.acl</code></td>
- <td>ACL for InterTrackerProtocol, used by the tasktrackers to
- communicate with the jobtracker.</td>
- </tr>
- <tr>
- <td><code>security.job.submission.protocol.acl</code></td>
- <td>ACL for JobSubmissionProtocol, used by job clients to
- communciate with the jobtracker for job submission, querying job status
- etc.</td>
- </tr>
- <tr>
- <td><code>security.task.umbilical.protocol.acl</code></td>
- <td>ACL for TaskUmbilicalProtocol, used by the map and reduce
- tasks to communicate with the parent tasktracker.</td>
- </tr>
- <tr>
<td><code>security.refresh.policy.protocol.acl</code></td>
<td>ACL for RefreshAuthorizationPolicyProtocol, used by the
dfsadmin and mradmin commands to refresh the security policy in-effect.
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1428602-1437112
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Tue Jan 22 19:33:02 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.security.authorize.Service;
/**
* This class contains constants for configuration keys used
@@ -114,7 +115,18 @@ public class CommonConfigurationKeys ext
SECURITY_HA_SERVICE_PROTOCOL_ACL = "security.ha.service.protocol.acl";
public static final String
SECURITY_ZKFC_PROTOCOL_ACL = "security.zkfc.protocol.acl";
-
+ public static final String
+ SECURITY_CLIENT_PROTOCOL_ACL = "security.client.protocol.acl";
+ public static final String SECURITY_CLIENT_DATANODE_PROTOCOL_ACL =
+ "security.client.datanode.protocol.acl";
+ public static final String
+ SECURITY_DATANODE_PROTOCOL_ACL = "security.datanode.protocol.acl";
+ public static final String
+ SECURITY_INTER_DATANODE_PROTOCOL_ACL = "security.inter.datanode.protocol.acl";
+ public static final String
+ SECURITY_NAMENODE_PROTOCOL_ACL = "security.namenode.protocol.acl";
+ public static final String SECURITY_QJOURNAL_SERVICE_PROTOCOL_ACL =
+ "security.qjournal.service.protocol.acl";
public static final String HADOOP_SECURITY_TOKEN_SERVICE_USE_IP =
"hadoop.security.token.service.use_ip";
public static final boolean HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT =
@@ -191,4 +203,4 @@ public class CommonConfigurationKeys ext
public static final long HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT =
4*60*60; // 4 hours
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Tue Jan 22 19:33:02 2013
@@ -173,6 +173,11 @@ public class CommonConfigurationKeysPubl
/** Default value for IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY */
public static final int IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT = 10000; // 10s
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String IPC_CLIENT_CONNECT_TIMEOUT_KEY =
+ "ipc.client.connect.timeout";
+ /** Default value for IPC_CLIENT_CONNECT_TIMEOUT_KEY */
+ public static final int IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT = 20000; // 20s
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String IPC_CLIENT_CONNECT_MAX_RETRIES_KEY =
"ipc.client.connect.max.retries";
/** Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY */
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Tue Jan 22 19:33:02 2013
@@ -172,7 +172,25 @@ import org.apache.hadoop.util.ShutdownHo
public final class FileContext {
public static final Log LOG = LogFactory.getLog(FileContext.class);
+ /**
+ * Default permission for directory and symlink
+ * In previous versions, this default permission was also used to
+ * create files, so files created end up with ugo+x permission.
+ * See HADOOP-9155 for detail.
+ * Two new constants are added to solve this, please use
+ * {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
+ * {@link FileContext#FILE_DEFAULT_PERM} for file.
+ * This constant is kept for compatibility.
+ */
public static final FsPermission DEFAULT_PERM = FsPermission.getDefault();
+ /**
+ * Default permission for directory
+ */
+ public static final FsPermission DIR_DEFAULT_PERM = FsPermission.getDirDefault();
+ /**
+ * Default permission for file
+ */
+ public static final FsPermission FILE_DEFAULT_PERM = FsPermission.getFileDefault();
/**
* Priority of the FileContext shutdown hook.
@@ -656,7 +674,7 @@ public final class FileContext {
CreateOpts.Perms permOpt =
(CreateOpts.Perms) CreateOpts.getOpt(CreateOpts.Perms.class, opts);
FsPermission permission = (permOpt != null) ? permOpt.getValue() :
- FsPermission.getDefault();
+ FILE_DEFAULT_PERM;
permission = permission.applyUMask(umask);
final CreateOpts[] updatedOpts =
@@ -704,7 +722,7 @@ public final class FileContext {
IOException {
final Path absDir = fixRelativePart(dir);
final FsPermission absFerms = (permission == null ?
- FsPermission.getDefault() : permission).applyUMask(umask);
+ FsPermission.getDirDefault() : permission).applyUMask(umask);
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
@@ -2157,7 +2175,7 @@ public final class FileContext {
FileStatus fs = FileContext.this.getFileStatus(qSrc);
if (fs.isDirectory()) {
checkDependencies(qSrc, qDst);
- mkdir(qDst, FsPermission.getDefault(), true);
+ mkdir(qDst, FsPermission.getDirDefault(), true);
FileStatus[] contents = listStatus(qSrc);
for (FileStatus content : contents) {
copy(makeQualified(content.getPath()), makeQualified(new Path(qDst,
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java Tue Jan 22 19:33:02 2013
@@ -79,8 +79,15 @@ public class FileStatus implements Writa
this.blocksize = blocksize;
this.modification_time = modification_time;
this.access_time = access_time;
- this.permission = (permission == null) ?
- FsPermission.getDefault() : permission;
+ if (permission != null) {
+ this.permission = permission;
+ } else if (isdir) {
+ this.permission = FsPermission.getDirDefault();
+ } else if (symlink!=null) {
+ this.permission = FsPermission.getDefault();
+ } else {
+ this.permission = FsPermission.getFileDefault();
+ }
this.owner = (owner == null) ? "" : owner;
this.group = (group == null) ? "" : group;
this.symlink = symlink;
@@ -217,7 +224,7 @@ public class FileStatus implements Writa
*/
protected void setPermission(FsPermission permission) {
this.permission = (permission == null) ?
- FsPermission.getDefault() : permission;
+ FsPermission.getFileDefault() : permission;
}
/**
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Tue Jan 22 19:33:02 2013
@@ -850,7 +850,7 @@ public abstract class FileSystem extends
long blockSize,
Progressable progress
) throws IOException {
- return this.create(f, FsPermission.getDefault().applyUMask(
+ return this.create(f, FsPermission.getFileDefault().applyUMask(
FsPermission.getUMask(getConf())), overwrite, bufferSize,
replication, blockSize, progress);
}
@@ -1030,7 +1030,7 @@ public abstract class FileSystem extends
boolean overwrite,
int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
- return this.createNonRecursive(f, FsPermission.getDefault(),
+ return this.createNonRecursive(f, FsPermission.getFileDefault(),
overwrite, bufferSize, replication, blockSize, progress);
}
@@ -1866,7 +1866,7 @@ public abstract class FileSystem extends
* Call {@link #mkdirs(Path, FsPermission)} with default permission.
*/
public boolean mkdirs(Path f) throws IOException {
- return mkdirs(f, FsPermission.getDefault());
+ return mkdirs(f, FsPermission.getDirDefault());
}
/**
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Tue Jan 22 19:33:02 2013
@@ -101,33 +101,98 @@ public class FileUtil {
* (4) If dir is a normal directory, then dir and all its contents recursively
* are deleted.
*/
- public static boolean fullyDelete(File dir) {
- if (dir.delete()) {
+ public static boolean fullyDelete(final File dir) {
+ return fullyDelete(dir, false);
+ }
+
+ /**
+ * Delete a directory and all its contents. If
+ * we return false, the directory may be partially-deleted.
+ * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+ * to by the symlink is not deleted.
+ * (2) If dir is symlink to a directory, symlink is deleted. The directory
+ * pointed to by symlink is not deleted.
+ * (3) If dir is a normal file, it is deleted.
+ * (4) If dir is a normal directory, then dir and all its contents recursively
+ * are deleted.
+ * @param dir the file or directory to be deleted
+ * @param tryGrantPermissions true if permissions should be modified to delete a file.
+ * @return true on success false on failure.
+ */
+ public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
+ if (tryGrantPermissions) {
+ // try to chmod +rwx the parent folder of the 'dir':
+ File parent = dir.getParentFile();
+ grantPermissions(parent);
+ }
+ if (deleteImpl(dir, false)) {
// dir is (a) normal file, (b) symlink to a file, (c) empty directory or
// (d) symlink to a directory
return true;
}
-
// handle nonempty directory deletion
- if (!fullyDeleteContents(dir)) {
+ if (!fullyDeleteContents(dir, tryGrantPermissions)) {
return false;
}
- return dir.delete();
+ return deleteImpl(dir, true);
+ }
+
+ /*
+ * Pure-Java implementation of "chmod +rwx f".
+ */
+ private static void grantPermissions(final File f) {
+ f.setExecutable(true);
+ f.setReadable(true);
+ f.setWritable(true);
}
+ private static boolean deleteImpl(final File f, final boolean doLog) {
+ if (f == null) {
+ LOG.warn("null file argument.");
+ return false;
+ }
+ final boolean wasDeleted = f.delete();
+ if (wasDeleted) {
+ return true;
+ }
+ final boolean ex = f.exists();
+ if (doLog && ex) {
+ LOG.warn("Failed to delete file or dir ["
+ + f.getAbsolutePath() + "]: it still exists.");
+ }
+ return !ex;
+ }
+
+ /**
+ * Delete the contents of a directory, not the directory itself. If
+ * we return false, the directory may be partially-deleted.
+ * If dir is a symlink to a directory, all the contents of the actual
+ * directory pointed to by dir will be deleted.
+ */
+ public static boolean fullyDeleteContents(final File dir) {
+ return fullyDeleteContents(dir, false);
+ }
+
/**
* Delete the contents of a directory, not the directory itself. If
* we return false, the directory may be partially-deleted.
* If dir is a symlink to a directory, all the contents of the actual
* directory pointed to by dir will be deleted.
+ * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
+ * and all the underlying directories before trying to delete their contents.
*/
- public static boolean fullyDeleteContents(File dir) {
+ public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
+ if (tryGrantPermissions) {
+ // to be able to list the dir and delete files from it
+ // we must grant the dir rwx permissions:
+ grantPermissions(dir);
+ }
boolean deletionSucceeded = true;
- File contents[] = dir.listFiles();
+ final File[] contents = dir.listFiles();
if (contents != null) {
for (int i = 0; i < contents.length; i++) {
if (contents[i].isFile()) {
- if (!contents[i].delete()) {// normal file or symlink to another file
+ if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
deletionSucceeded = false;
continue; // continue deletion of other files/dirs under dir
}
@@ -135,16 +200,16 @@ public class FileUtil {
// Either directory or symlink to another directory.
// Try deleting the directory as this might be a symlink
boolean b = false;
- b = contents[i].delete();
+ b = deleteImpl(contents[i], false);
if (b){
//this was indeed a symlink or an empty directory
continue;
}
// if not an empty directory or symlink let
// fullydelete handle it.
- if (!fullyDelete(contents[i])) {
+ if (!fullyDelete(contents[i], tryGrantPermissions)) {
deletionSucceeded = false;
- continue; // continue deletion of other files/dirs under dir
+ // continue deletion of other files/dirs under dir
}
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java Tue Jan 22 19:33:02 2013
@@ -224,7 +224,7 @@ public class FTPFileSystem extends FileS
}
Path parent = absolute.getParent();
- if (parent == null || !mkdirs(client, parent, FsPermission.getDefault())) {
+ if (parent == null || !mkdirs(client, parent, FsPermission.getDirDefault())) {
parent = (parent == null) ? new Path("/") : parent;
disconnect(client);
throw new IOException("create(): Mkdirs failed to create: " + parent);
@@ -484,7 +484,7 @@ public class FTPFileSystem extends FileS
if (!exists(client, absolute)) {
Path parent = absolute.getParent();
created = (parent == null || mkdirs(client, parent, FsPermission
- .getDefault()));
+ .getDirDefault()));
if (created) {
String parentDir = parent.toUri().getPath();
client.changeWorkingDirectory(parentDir);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java Tue Jan 22 19:33:02 2013
@@ -85,7 +85,7 @@ public class RawLocalFs extends Delegate
"system: "+target.toString());
}
if (createParent) {
- mkdir(link.getParent(), FsPermission.getDefault(), true);
+ mkdir(link.getParent(), FsPermission.getDirDefault(), true);
}
// NB: Use createSymbolicLink in java.nio.file.Path once available
try {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java Tue Jan 22 19:33:02 2013
@@ -275,12 +275,35 @@ public class FsPermission implements Wri
conf.setInt(DEPRECATED_UMASK_LABEL, umask.toShort());
}
- /** Get the default permission. */
+ /**
+ * Get the default permission for directory and symlink.
+ * In previous versions, this default permission was also used to
+ * create files, so files created end up with ugo+x permission.
+ * See HADOOP-9155 for detail.
+ * Two new methods are added to solve this, please use
+ * {@link FsPermission#getDirDefault()} for directory, and use
+ * {@link FsPermission#getFileDefault()} for file.
+ * This method is kept for compatibility.
+ */
public static FsPermission getDefault() {
return new FsPermission((short)00777);
}
/**
+ * Get the default permission for directory.
+ */
+ public static FsPermission getDirDefault() {
+ return new FsPermission((short)00777);
+ }
+
+ /**
+ * Get the default permission for file.
+ */
+ public static FsPermission getFileDefault() {
+ return new FsPermission((short)00666);
+ }
+
+ /**
* Create a FsPermission from a Unix symbolic permission string
* @param unixSymbolicPermission e.g. "-rw-rw-rw-"
*/
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java Tue Jan 22 19:33:02 2013
@@ -613,7 +613,7 @@ public class ActiveStandbyElector implem
// Unfortunately, the ZooKeeper constructor connects to ZooKeeper and
// may trigger the Connected event immediately. So, if we register the
// watcher after constructing ZooKeeper, we may miss that event. Instead,
- // we construct the watcher first, and have it queue any events it receives
+ // we construct the watcher first, and have it block any events it receives
// before we can set its ZooKeeper reference.
WatcherWithClientRef watcher = new WatcherWithClientRef();
ZooKeeper zk = new ZooKeeper(zkHostPort, zkSessionTimeout, watcher);
@@ -1002,19 +1002,17 @@ public class ActiveStandbyElector implem
private CountDownLatch hasReceivedEvent = new CountDownLatch(1);
/**
- * If any events arrive before the reference to ZooKeeper is set,
- * they get queued up and later forwarded when the reference is
- * available.
+ * Latch used to wait until the reference to ZooKeeper is set.
*/
- private final List<WatchedEvent> queuedEvents = Lists.newLinkedList();
+ private CountDownLatch hasSetZooKeeper = new CountDownLatch(1);
private WatcherWithClientRef() {
}
private WatcherWithClientRef(ZooKeeper zk) {
- this.zk = zk;
+ setZooKeeperRef(zk);
}
-
+
/**
* Waits for the next event from ZooKeeper to arrive.
*
@@ -1029,9 +1027,7 @@ public class ActiveStandbyElector implem
if (!hasReceivedEvent.await(connectionTimeoutMs, TimeUnit.MILLISECONDS)) {
LOG.error("Connection timed out: couldn't connect to ZooKeeper in "
+ connectionTimeoutMs + " milliseconds");
- synchronized (this) {
- zk.close();
- }
+ zk.close();
throw KeeperException.create(Code.CONNECTIONLOSS);
}
} catch (InterruptedException e) {
@@ -1041,29 +1037,18 @@ public class ActiveStandbyElector implem
}
}
- private synchronized void setZooKeeperRef(ZooKeeper zk) {
+ private void setZooKeeperRef(ZooKeeper zk) {
Preconditions.checkState(this.zk == null,
"zk already set -- must be set exactly once");
this.zk = zk;
-
- for (WatchedEvent e : queuedEvents) {
- forwardEvent(e);
- }
- queuedEvents.clear();
+ hasSetZooKeeper.countDown();
}
@Override
- public synchronized void process(WatchedEvent event) {
- if (zk != null) {
- forwardEvent(event);
- } else {
- queuedEvents.add(event);
- }
- }
-
- private void forwardEvent(WatchedEvent event) {
+ public void process(WatchedEvent event) {
hasReceivedEvent.countDown();
try {
+ hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
ActiveStandbyElector.this.processWatchEvent(
zk, event);
} catch (Throwable t) {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Tue Jan 22 19:33:02 2013
@@ -246,6 +246,7 @@ public class HttpServer implements Filte
// default value (currently 250).
QueuedThreadPool threadPool = maxThreads == -1 ?
new QueuedThreadPool() : new QueuedThreadPool(maxThreads);
+ threadPool.setDaemon(true);
webServer.setThreadPool(threadPool);
final String appDir = getWebAppsPath(name);
@@ -312,6 +313,7 @@ public class HttpServer implements Filte
// the same port with indeterminate routing of incoming requests to them
ret.setReuseAddress(false);
}
+ ret.setHeaderBufferSize(1024*64);
return ret;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java Tue Jan 22 19:33:02 2013
@@ -122,7 +122,7 @@ public class CompressionCodecFactory {
if (codecsString != null) {
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
- String codecSubstring = codecSplit.nextToken();
+ String codecSubstring = codecSplit.nextToken().trim();
if (codecSubstring.length() != 0) {
try {
Class<?> cls = conf.getClassByName(codecSubstring);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java Tue Jan 22 19:33:02 2013
@@ -40,14 +40,74 @@ public class GzipCodec extends DefaultCo
protected static class GzipOutputStream extends CompressorStream {
private static class ResetableGZIPOutputStream extends GZIPOutputStream {
-
+ private static final int TRAILER_SIZE = 8;
+ public static final String JVMVendor= System.getProperty("java.vendor");
+ public static final String JVMVersion= System.getProperty("java.version");
+ private static final boolean HAS_BROKEN_FINISH =
+ (JVMVendor.contains("IBM") && JVMVersion.contains("1.6.0"));
+
public ResetableGZIPOutputStream(OutputStream out) throws IOException {
super(out);
}
-
+
public void resetState() throws IOException {
def.reset();
}
+
+ /**
+ * Override this method for HADOOP-8419.
+ * Override because IBM implementation calls def.end() which
+ * causes problem when reseting the stream for reuse.
+ *
+ */
+ @Override
+ public void finish() throws IOException {
+ if (HAS_BROKEN_FINISH) {
+ if (!def.finished()) {
+ def.finish();
+ while (!def.finished()) {
+ int i = def.deflate(this.buf, 0, this.buf.length);
+ if ((def.finished()) && (i <= this.buf.length - TRAILER_SIZE)) {
+ writeTrailer(this.buf, i);
+ i += TRAILER_SIZE;
+ out.write(this.buf, 0, i);
+
+ return;
+ }
+ if (i > 0) {
+ out.write(this.buf, 0, i);
+ }
+ }
+
+ byte[] arrayOfByte = new byte[TRAILER_SIZE];
+ writeTrailer(arrayOfByte, 0);
+ out.write(arrayOfByte);
+ }
+ } else {
+ super.finish();
+ }
+ }
+
+ /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
+ private void writeTrailer(byte[] paramArrayOfByte, int paramInt)
+ throws IOException {
+ writeInt((int)this.crc.getValue(), paramArrayOfByte, paramInt);
+ writeInt(this.def.getTotalIn(), paramArrayOfByte, paramInt + 4);
+ }
+
+ /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
+ private void writeInt(int paramInt1, byte[] paramArrayOfByte, int paramInt2)
+ throws IOException {
+ writeShort(paramInt1 & 0xFFFF, paramArrayOfByte, paramInt2);
+ writeShort(paramInt1 >> 16 & 0xFFFF, paramArrayOfByte, paramInt2 + 2);
+ }
+
+ /** re-implement for HADOOP-8419 because the relative method in jdk is invisible */
+ private void writeShort(int paramInt1, byte[] paramArrayOfByte, int paramInt2)
+ throws IOException {
+ paramArrayOfByte[paramInt2] = (byte)(paramInt1 & 0xFF);
+ paramArrayOfByte[(paramInt2 + 1)] = (byte)(paramInt1 >> 8 & 0xFF);
+ }
}
public GzipOutputStream(OutputStream out) throws IOException {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java Tue Jan 22 19:33:02 2013
@@ -106,6 +106,8 @@ public class Client {
private SocketFactory socketFactory; // how to create sockets
private int refCount = 1;
+
+ private final int connectionTimeout;
final static int PING_CALL_ID = -1;
@@ -159,7 +161,16 @@ public class Client {
}
return -1;
}
-
+ /**
+ * set the connection timeout value in configuration
+ *
+ * @param conf Configuration
+ * @param timeout the socket connect timeout value
+ */
+ public static final void setConnectTimeout(Configuration conf, int timeout) {
+ conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY, timeout);
+ }
+
/**
* Increment this client's reference count
*
@@ -494,8 +505,7 @@ public class Client {
}
}
- // connection time out is 20s
- NetUtils.connect(this.socket, server, 20000);
+ NetUtils.connect(this.socket, server, connectionTimeout);
if (rpcTimeout > 0) {
pingInterval = rpcTimeout; // rpcTimeout overwrites pingInterval
}
@@ -1034,6 +1044,8 @@ public class Client {
this.valueClass = valueClass;
this.conf = conf;
this.socketFactory = factory;
+ this.connectionTimeout = conf.getInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT);
}
/**
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java Tue Jan 22 19:33:02 2013
@@ -25,6 +25,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.NetworkInterface;
import java.net.NoRouteToHostException;
+import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.SocketException;
@@ -865,4 +866,23 @@ public class NetUtils {
}
return addrs;
}
+
+ /**
+ * Return a free port number. There is no guarantee it will remain free, so
+ * it should be used immediately.
+ *
+ * @returns A free port for binding a local socket
+ */
+ public static int getFreeSocketPort() {
+ int port = 0;
+ try {
+ ServerSocket s = new ServerSocket(0);
+ port = s.getLocalPort();
+ s.close();
+ return port;
+ } catch (IOException e) {
+ // Could not get a free port. Return default port 0.
+ }
+ return port;
+ }
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java Tue Jan 22 19:33:02 2013
@@ -18,10 +18,13 @@
package org.apache.hadoop.security;
+import java.io.BufferedInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -148,8 +151,32 @@ public class Credentials implements Writ
in.close();
return credentials;
} catch(IOException ioe) {
+ throw new IOException("Exception reading " + filename, ioe);
+ } finally {
IOUtils.cleanup(LOG, in);
+ }
+ }
+
+ /**
+ * Convenience method for reading a token storage file, and loading the Tokens
+ * therein in the passed UGI
+ * @param filename
+ * @param conf
+ * @throws IOException
+ */
+ public static Credentials readTokenStorageFile(File filename, Configuration conf)
+ throws IOException {
+ DataInputStream in = null;
+ Credentials credentials = new Credentials();
+ try {
+ in = new DataInputStream(new BufferedInputStream(
+ new FileInputStream(filename)));
+ credentials.readTokenStorageStream(in);
+ return credentials;
+ } catch(IOException ioe) {
throw new IOException("Exception reading " + filename, ioe);
+ } finally {
+ IOUtils.cleanup(LOG, in);
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java Tue Jan 22 19:33:02 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.security;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT;
+import java.io.File;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.security.AccessControlContext;
@@ -656,10 +657,11 @@ public class UserGroupInformation {
String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
if (fileLocation != null) {
- // load the token storage file and put all of the tokens into the
- // user.
+ // Load the token storage file and put all of the tokens into the
+ // user. Don't use the FileSystem API for reading since it has a lock
+ // cycle (HADOOP-9212).
Credentials cred = Credentials.readTokenStorageFile(
- new Path("file:///" + fileLocation), conf);
+ new File(fileLocation), conf);
loginUser.addCredentials(cred);
}
loginUser.spawnAutoRenewalThreadForUserCreds();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java Tue Jan 22 19:33:02 2013
@@ -27,7 +27,7 @@ import org.apache.hadoop.classification.
* This class is thread safe.
*/
@InterfaceAudience.Private
-public abstract class SequentialNumber {
+public abstract class SequentialNumber implements IdGenerator {
private final AtomicLong currentValue;
/** Create a new instance with the given initial value. */
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo Tue Jan 22 19:33:02 2013
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.security.AnnotatedSecurityInfo
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Tue Jan 22 19:33:02 2013
@@ -80,9 +80,17 @@
<property>
<name>hadoop.security.group.mapping</name>
- <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
+ <value>org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback</value>
<description>
- Class for user to group mapping (get groups for a given user) for ACL
+ Class for user to group mapping (get groups for a given user) for ACL.
+ The default implementation,
+ org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback,
+ will determine if the Java Native Interface (JNI) is available. If JNI is
+ available the implementation will use the API within hadoop to resolve a
+ list of groups for a user. If JNI is not available then the shell
+ implementation, ShellBasedUnixGroupsMapping, is used. This implementation
+ shells out to the Linux/Unix environment with the
+ <code>bash -c groups</code> command to resolve a list of groups for a user.
</description>
</property>
@@ -566,6 +574,14 @@
</property>
<property>
+ <name>ipc.client.connect.timeout</name>
+ <value>20000</value>
+ <description>Indicates the number of milliseconds a client will wait for the
+ socket to establish a server connection.
+ </description>
+</property>
+
+<property>
<name>ipc.client.connect.max.retries.on.timeouts</name>
<value>45</value>
<description>Indicates the number of retries a client will make on socket timeout
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1428602-1437112
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextPermissionBase.java Tue Jan 22 19:33:02 2013
@@ -95,7 +95,7 @@ public abstract class FileContextPermiss
String filename = "foo";
Path f = getTestRootPath(fc, filename);
createFile(fc, filename);
- doFilePermissionCheck(FileContext.DEFAULT_PERM.applyUMask(fc.getUMask()),
+ doFilePermissionCheck(FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask()),
fc.getFileStatus(f).getPermission());
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java Tue Jan 22 19:33:02 2013
@@ -23,12 +23,9 @@ import java.io.IOException;
import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
/**
@@ -45,15 +42,13 @@ import org.apache.hadoop.fs.permission.F
* </p>
*/
public abstract class FileSystemContractBaseTest extends TestCase {
+ private static final Log LOG =
+ LogFactory.getLog(FileSystemContractBaseTest.class);
+
protected final static String TEST_UMASK = "062";
protected FileSystem fs;
- protected byte[] data = new byte[getBlockSize() * 2]; // two blocks of data
- {
- for (int i = 0; i < data.length; i++) {
- data[i] = (byte) (i % 10);
- }
- }
-
+ protected byte[] data = dataset(getBlockSize() * 2, 0, 255);
+
@Override
protected void tearDown() throws Exception {
fs.delete(path("/test"), true);
@@ -235,35 +230,16 @@ public abstract class FileSystemContract
public void testWriteReadAndDeleteTwoBlocks() throws Exception {
writeReadAndDelete(getBlockSize() * 2);
}
-
+
+ /**
+ * Write a dataset, read it back in and verify that they match.
+ * Afterwards, the file is deleted.
+ * @param len length of data
+ * @throws IOException on IO failures
+ */
protected void writeReadAndDelete(int len) throws IOException {
Path path = path("/test/hadoop/file");
-
- fs.mkdirs(path.getParent());
-
- FSDataOutputStream out = fs.create(path, false,
- fs.getConf().getInt("io.file.buffer.size", 4096),
- (short) 1, getBlockSize());
- out.write(data, 0, len);
- out.close();
-
- assertTrue("Exists", fs.exists(path));
- assertEquals("Length", len, fs.getFileStatus(path).getLen());
-
- FSDataInputStream in = fs.open(path);
- byte[] buf = new byte[len];
- in.readFully(0, buf);
- in.close();
-
- assertEquals(len, buf.length);
- for (int i = 0; i < buf.length; i++) {
- assertEquals("Position " + i, data[i], buf[i]);
- }
-
- assertTrue("Deleted", fs.delete(path, false));
-
- assertFalse("No longer exists", fs.exists(path));
-
+ writeAndRead(path, data, len, false, true);
}
public void testOverwrite() throws IOException {
@@ -494,4 +470,147 @@ public abstract class FileSystemContract
assertEquals("Source exists", srcExists, fs.exists(src));
assertEquals("Destination exists", dstExists, fs.exists(dst));
}
+
+ /**
+ * Verify that if you take an existing file and overwrite it, the new values
+ * get picked up.
+ * This is a test for the behavior of eventually consistent
+ * filesystems.
+ *
+ * @throws Exception on any failure
+ */
+
+ public void testOverWriteAndRead() throws Exception {
+ int blockSize = getBlockSize();
+
+ byte[] filedata1 = dataset(blockSize * 2, 'A', 26);
+ byte[] filedata2 = dataset(blockSize * 2, 'a', 26);
+ Path path = path("/test/hadoop/file-overwrite");
+ writeAndRead(path, filedata1, blockSize, true, false);
+ writeAndRead(path, filedata2, blockSize, true, false);
+ writeAndRead(path, filedata1, blockSize * 2, true, false);
+ writeAndRead(path, filedata2, blockSize * 2, true, false);
+ writeAndRead(path, filedata1, blockSize, true, false);
+ writeAndRead(path, filedata2, blockSize * 2, true, false);
+ }
+
+ /**
+ *
+ * Write a file and read it in, validating the result. Optional flags control
+ * whether file overwrite operations should be enabled, and whether the
+ * file should be deleted afterwards.
+ *
+ * If there is a mismatch between what was written and what was expected,
+ * a small range of bytes either side of the first error are logged to aid
+ * diagnosing what problem occurred -whether it was a previous file
+ * or a corrupting of the current file. This assumes that two
+ * sequential runs to the same path use datasets with different character
+ * moduli.
+ *
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @param delete should the file be deleted afterwards? -with a verification
+ * that it worked. Deletion is not attempted if an assertion has failed
+ * earlier -it is not in a <code>finally{}</code> block.
+ * @throws IOException IO problems
+ */
+ protected void writeAndRead(Path path, byte[] src, int len,
+ boolean overwrite,
+ boolean delete) throws IOException {
+ assertTrue("Not enough data in source array to write " + len + " bytes",
+ src.length >= len);
+ fs.mkdirs(path.getParent());
+
+ FSDataOutputStream out = fs.create(path, overwrite,
+ fs.getConf().getInt("io.file.buffer.size",
+ 4096),
+ (short) 1, getBlockSize());
+ out.write(src, 0, len);
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+ assertEquals("Length", len, fs.getFileStatus(path).getLen());
+
+ FSDataInputStream in = fs.open(path);
+ byte[] buf = new byte[len];
+ in.readFully(0, buf);
+ in.close();
+
+ assertEquals(len, buf.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (src[i] != buf[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ LOG.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = buf[i];
+ byte expected = src[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ LOG.warn(line);
+ }
+ fail(message);
+ }
+
+ if (delete) {
+ boolean deleted = fs.delete(path, false);
+ assertTrue("Deleted", deleted);
+ assertFalse("No longer exists", fs.exists(path));
+ }
+ }
+
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ protected String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ protected byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java Tue Jan 22 19:33:02 2013
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java Tue Jan 22 19:33:02 2013
@@ -121,7 +121,7 @@ public class TestFileStatus {
FileStatus fileStatus = new FileStatus(LENGTH, isdir,
REPLICATION, BLKSIZE, MTIME, PATH);
validateAccessors(fileStatus, LENGTH, isdir, REPLICATION, BLKSIZE, MTIME,
- 0, FsPermission.getDefault(), "", "", null, PATH);
+ 0, FsPermission.getDirDefault(), "", "", null, PATH);
}
/**
@@ -131,7 +131,7 @@ public class TestFileStatus {
public void constructorBlank() throws IOException {
FileStatus fileStatus = new FileStatus();
validateAccessors(fileStatus, 0, false, 0, 0, 0,
- 0, FsPermission.getDefault(), "", "", null, null);
+ 0, FsPermission.getFileDefault(), "", "", null, null);
}
/**
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java Tue Jan 22 19:33:02 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs;
+import org.junit.Before;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -176,12 +177,26 @@ public class TestFileUtil {
//Expected an IOException
}
}
+
+ @Before
+ public void before() throws IOException {
+ cleanupImpl();
+ }
@After
public void tearDown() throws IOException {
- FileUtil.fullyDelete(del);
- FileUtil.fullyDelete(tmp);
- FileUtil.fullyDelete(partitioned);
+ cleanupImpl();
+ }
+
+ private void cleanupImpl() throws IOException {
+ FileUtil.fullyDelete(del, true);
+ Assert.assertTrue(!del.exists());
+
+ FileUtil.fullyDelete(tmp, true);
+ Assert.assertTrue(!tmp.exists());
+
+ FileUtil.fullyDelete(partitioned, true);
+ Assert.assertTrue(!partitioned.exists());
}
@Test
@@ -272,12 +287,14 @@ public class TestFileUtil {
Assert.assertTrue(new File(tmp, FILE).exists());
}
- private File xSubDir = new File(del, "xsubdir");
- private File ySubDir = new File(del, "ysubdir");
- static String file1Name = "file1";
- private File file2 = new File(xSubDir, "file2");
- private File file3 = new File(ySubDir, "file3");
- private File zlink = new File(del, "zlink");
+ private final File xSubDir = new File(del, "xSubDir");
+ private final File xSubSubDir = new File(xSubDir, "xSubSubDir");
+ private final File ySubDir = new File(del, "ySubDir");
+ private static final String file1Name = "file1";
+ private final File file2 = new File(xSubDir, "file2");
+ private final File file22 = new File(xSubSubDir, "file22");
+ private final File file3 = new File(ySubDir, "file3");
+ private final File zlink = new File(del, "zlink");
/**
* Creates a directory which can not be deleted completely.
@@ -289,10 +306,14 @@ public class TestFileUtil {
* |
* .---------------------------------------,
* | | | |
- * file1(!w) xsubdir(-w) ysubdir(+w) zlink
- * | |
- * file2 file3
- *
+ * file1(!w) xSubDir(-rwx) ySubDir(+w) zlink
+ * | | |
+ * | file2(-rwx) file3
+ * |
+ * xSubSubDir(-rwx)
+ * |
+ * file22(-rwx)
+ *
* @throws IOException
*/
private void setupDirsAndNonWritablePermissions() throws IOException {
@@ -305,7 +326,16 @@ public class TestFileUtil {
xSubDir.mkdirs();
file2.createNewFile();
- xSubDir.setWritable(false);
+
+ xSubSubDir.mkdirs();
+ file22.createNewFile();
+
+ revokePermissions(file22);
+ revokePermissions(xSubSubDir);
+
+ revokePermissions(file2);
+ revokePermissions(xSubDir);
+
ySubDir.mkdirs();
file3.createNewFile();
@@ -317,23 +347,43 @@ public class TestFileUtil {
FileUtil.symLink(tmpFile.toString(), zlink.toString());
}
+ private static void grantPermissions(final File f) {
+ f.setReadable(true);
+ f.setWritable(true);
+ f.setExecutable(true);
+ }
+
+ private static void revokePermissions(final File f) {
+ f.setWritable(false);
+ f.setExecutable(false);
+ f.setReadable(false);
+ }
+
// Validates the return value.
- // Validates the existence of directory "xsubdir" and the file "file1"
- // Sets writable permissions for the non-deleted dir "xsubdir" so that it can
- // be deleted in tearDown().
- private void validateAndSetWritablePermissions(boolean ret) {
- xSubDir.setWritable(true);
- Assert.assertFalse("The return value should have been false!", ret);
- Assert.assertTrue("The file file1 should not have been deleted!",
+ // Validates the existence of the file "file1"
+ private void validateAndSetWritablePermissions(
+ final boolean expectedRevokedPermissionDirsExist, final boolean ret) {
+ grantPermissions(xSubDir);
+ grantPermissions(xSubSubDir);
+
+ Assert.assertFalse("The return value should have been false.", ret);
+ Assert.assertTrue("The file file1 should not have been deleted.",
new File(del, file1Name).exists());
- Assert.assertTrue(
- "The directory xsubdir should not have been deleted!",
- xSubDir.exists());
- Assert.assertTrue("The file file2 should not have been deleted!",
- file2.exists());
- Assert.assertFalse("The directory ysubdir should have been deleted!",
+
+ Assert.assertEquals(
+ "The directory xSubDir *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, xSubDir.exists());
+ Assert.assertEquals("The file file2 *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, file2.exists());
+ Assert.assertEquals(
+ "The directory xSubSubDir *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, xSubSubDir.exists());
+ Assert.assertEquals("The file file22 *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, file22.exists());
+
+ Assert.assertFalse("The directory ySubDir should have been deleted.",
ySubDir.exists());
- Assert.assertFalse("The link zlink should have been deleted!",
+ Assert.assertFalse("The link zlink should have been deleted.",
zlink.exists());
}
@@ -346,7 +396,15 @@ public class TestFileUtil {
LOG.info("Running test to verify failure of fullyDelete()");
setupDirsAndNonWritablePermissions();
boolean ret = FileUtil.fullyDelete(new MyFile(del));
- validateAndSetWritablePermissions(ret);
+ validateAndSetWritablePermissions(true, ret);
+ }
+
+ @Test
+ public void testFailFullyDeleteGrantPermissions() throws IOException {
+ setupDirsAndNonWritablePermissions();
+ boolean ret = FileUtil.fullyDelete(new MyFile(del), true);
+ // this time the directories with revoked permissions *should* be deleted:
+ validateAndSetWritablePermissions(false, ret);
}
/**
@@ -395,7 +453,10 @@ public class TestFileUtil {
*/
@Override
public File[] listFiles() {
- File[] files = super.listFiles();
+ final File[] files = super.listFiles();
+ if (files == null) {
+ return null;
+ }
List<File> filesList = Arrays.asList(files);
Collections.sort(filesList);
File[] myFiles = new MyFile[files.length];
@@ -416,10 +477,18 @@ public class TestFileUtil {
LOG.info("Running test to verify failure of fullyDeleteContents()");
setupDirsAndNonWritablePermissions();
boolean ret = FileUtil.fullyDeleteContents(new MyFile(del));
- validateAndSetWritablePermissions(ret);
+ validateAndSetWritablePermissions(true, ret);
}
@Test
+ public void testFailFullyDeleteContentsGrantPermissions() throws IOException {
+ setupDirsAndNonWritablePermissions();
+ boolean ret = FileUtil.fullyDeleteContents(new MyFile(del), true);
+ // this time the directories with revoked permissions *should* be deleted:
+ validateAndSetWritablePermissions(false, ret);
+ }
+
+ @Test
public void testCopyMergeSingleDirectory() throws IOException {
setupDirs();
boolean copyMergeResult = copyMerge("partitioned", "tmp/merged");
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFSFileContextMainOperations.java Tue Jan 22 19:33:02 2013
@@ -24,6 +24,8 @@ import org.apache.hadoop.conf.Configurat
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.apache.hadoop.fs.FileContextTestHelper;
+import org.apache.hadoop.fs.permission.FsPermission;
public class TestLocalFSFileContextMainOperations extends FileContextMainOperationsBaseTest {
@@ -47,4 +49,14 @@ public class TestLocalFSFileContextMainO
FileContext fc1 = FileContext.getLocalFSFileContext();
Assert.assertTrue(fc1 != fc);
}
+
+ @Test
+ public void testDefaultFilePermission() throws IOException {
+ Path file = FileContextTestHelper.getTestRootPath(fc,
+ "testDefaultFilePermission");
+ FileContextTestHelper.createFile(fc, file);
+ FsPermission expect = FileContext.FILE_DEFAULT_PERM.applyUMask(fc.getUMask());
+ Assert.assertEquals(expect, fc.getFileStatus(file)
+ .getPermission());
+ }
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java Tue Jan 22 19:33:02 2013
@@ -73,7 +73,7 @@ public class TestLocalFileSystemPermissi
try {
FsPermission initialPermission = getPermission(localfs, f);
System.out.println(filename + ": " + initialPermission);
- assertEquals(FsPermission.getDefault().applyUMask(FsPermission.getUMask(conf)), initialPermission);
+ assertEquals(FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)), initialPermission);
}
catch(Exception e) {
System.out.println(StringUtils.stringifyException(e));