You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2014/01/31 04:24:28 UTC
svn commit: r1563042 [1/2] - in
/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/qjournal/server/
src/main/java/org/apache/hadoop/hdfs/server...
Author: szetszwo
Date: Fri Jan 31 03:24:24 2014
New Revision: 1563042
URL: http://svn.apache.org/r1563042
Log:
Merge r1555021 through r1563041 from trunk.
Modified:
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java
hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1561944-1563041
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jan 31 03:24:24 2014
@@ -292,6 +292,30 @@ Release 2.4.0 - UNRELEASED
NEW FEATURES
+ IMPROVEMENTS
+
+ HDFS-5781. Use an array to record the mapping between FSEditLogOpCode and
+ the corresponding byte value. (jing9)
+
+ OPTIMIZATIONS
+
+ HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery
+ (todd)
+
+ BUG FIXES
+
+ HDFS-5492. Port HDFS-2069 (Incorrect default trash interval in the
+ docs) to trunk. (Akira Ajisaka via Arpit Agarwal)
+
+ HDFS-5843. DFSClient.getFileChecksum() throws IOException if checksum is
+ disabled. (Laurent Goujon via jing9)
+
+Release 2.3.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
HDFS-5122. Support failover and retry in WebHdfsFileSystem for NN HA.
(Haohui Mai via jing9)
@@ -329,6 +353,43 @@ Release 2.4.0 - UNRELEASED
IMPROVEMENTS
+ HDFS-5360. Improvement of usage message of renameSnapshot and
+ deleteSnapshot. (Shinichi Yamashita via wang)
+
+ HDFS-5331. make SnapshotDiff.java to a o.a.h.util.Tool interface implementation.
+ (Vinayakumar B via umamahesh)
+
+ HDFS-4657. Limit the number of blocks logged by the NN after a block
+ report to a configurable value. (Aaron T. Myers via Colin Patrick
+ McCabe)
+
+ HDFS-5344. Make LsSnapshottableDir as Tool interface implementation. (Sathish via umamahesh)
+
+ HDFS-5544. Adding Test case For Checking dfs.checksum type as NULL value. (Sathish via umamahesh)
+
+ HDFS-5568. Support includeSnapshots option with Fsck command. (Vinayakumar B via umamahesh)
+
+ HDFS-4983. Numeric usernames do not work with WebHDFS FS. (Yongjun Zhang via
+ jing9)
+
+ HDFS-5592. statechangeLog of completeFile should be logged only in case of success.
+ (Vinayakumar via umamahesh)
+
+ HDFS-5662. Can't decommission a DataNode due to file's replication factor
+ larger than the rest of the cluster size. (brandonli)
+
+ HDFS-5068. Convert NNThroughputBenchmark to a Tool to allow generic options.
+ (shv)
+
+ HDFS-5675. Add Mkdirs operation to NNThroughputBenchmark.
+ (Plamen Jeliazkov via shv)
+
+ HDFS-5677. Need error checking for HA cluster configuration.
+ (Vincent Sheffer via cos)
+
+ HDFS-5825. Use FileUtils.copyFile() to implement DFSTestUtils.copyFile().
+ (Haohui Mai via Arpit Agarwal)
+
HDFS-5267. Remove volatile from LightWeightHashSet. (Junping Du via llu)
HDFS-4278. Log an ERROR when DFS_BLOCK_ACCESS_TOKEN_ENABLE config is
@@ -504,8 +565,7 @@ Release 2.4.0 - UNRELEASED
HDFS-5788. listLocatedStatus response can be very large. (Nathan Roberts
via kihwal)
- HDFS-5781. Use an array to record the mapping between FSEditLogOpCode and
- the corresponding byte value. (jing9)
+ HDFS-5841. Update HDFS caching documentation with new changes. (wang)
OPTIMIZATIONS
@@ -521,6 +581,177 @@ Release 2.4.0 - UNRELEASED
BUG FIXES
+ HDFS-5307. Support both HTTP and HTTPS in jsp pages (Haohui Mai via
+ brandonli)
+
+ HDFS-5291. Standby namenode after transition to active goes into safemode.
+ (jing9)
+
+ HDFS-5317. Go back to DFS Home link does not work on datanode webUI
+ (Haohui Mai via brandonli)
+
+ HDFS-5316. Namenode ignores the default https port (Haohui Mai via
+ brandonli)
+
+ HDFS-5281. COMMIT request should not block. (brandonli)
+
+ HDFS-5337. should do hsync for a commit request even there is no pending
+ writes (brandonli)
+
+ HDFS-5335. Hive query failed with possible race in dfs output stream.
+ (Haohui Mai via suresh)
+
+ HDFS-5322. HDFS delegation token not found in cache errors seen on secure HA
+ clusters. (jing9)
+
+ HDFS-5329. Update FSNamesystem#getListing() to handle inode path in startAfter
+ token. (brandonli)
+
+ HDFS-5330. fix readdir and readdirplus for large directories (brandonli)
+
+ HDFS-5370. Typo in Error Message: different between range in condition
+ and range in error message. (Kousuke Saruta via suresh)
+
+ HDFS-5365. Fix libhdfs compile error on FreeBSD9. (Radim Kolar via cnauroth)
+
+ HDFS-5347. Add HDFS NFS user guide. (brandonli)
+
+ HDFS-5403. WebHdfs client cannot communicate with older WebHdfs servers
+ post HDFS-5306. (atm)
+
+ HDFS-5171. NFS should create input stream for a file and try to share it
+ with multiple read requests. (Haohui Mai via brandonli)
+
+ HDFS-5413. hdfs.cmd does not support passthrough to any arbitrary class.
+ (cnauroth)
+
+ HDFS-5433. When reloading fsimage during checkpointing, we should clear
+ existing snapshottable directories. (Aaron T. Myers via wang)
+
+ HDFS-5432. TestDatanodeJsp fails on Windows due to assumption that loopback
+ address resolves to host name localhost. (cnauroth)
+
+ HDFS-5065. TestSymlinkHdfsDisable fails on Windows. (ivanmi)
+
+ HDFS-4633 TestDFSClientExcludedNodes fails sporadically if excluded nodes
+ cache expires too quickly (Chris Nauroth via Sanjay)
+
+ HDFS-5037. Active NN should trigger its own edit log rolls (wang)
+
+ HDFS-5035. getFileLinkStatus and rename do not correctly check permissions
+ of symlinks. (Andrew Wang via Colin Patrick McCabe)
+
+ HDFS-5456. NameNode startup progress creates new steps if caller attempts to
+ create a counter for a step that doesn't already exist. (cnauroth)
+
+ HDFS-5458. Datanode failed volume threshold ignored if exception is thrown
+ in getDataDirsFromURIs. (Mike Mellenthin via wang)
+
+ HDFS-5252. Stable write is not handled correctly in someplace. (brandonli)
+
+ HDFS-5364. Add OpenFileCtx cache. (brandonli)
+
+ HDFS-5469. Add configuration property for the sub-directroy export path
+ (brandonli)
+
+ HDFS-5519. COMMIT handler should update the commit status after sync
+ (brandonli)
+
+ HDFS-5372. In FSNamesystem, hasReadLock() returns false if the current thread
+ holds the write lock (VinayaKumar B via umamahesh)
+
+ HDFS-4516. Client crash after block allocation and NN switch before lease recovery for
+ the same file can cause readers to fail forever (VinaayKumar B via umamahesh)
+
+ HDFS-5014. Process register commands with out holding BPOfferService lock.
+ (Vinaykumar B via umamahesh)
+
+ HDFS-5288. Close idle connections in portmap (Haohui Mai via brandonli)
+
+ HDFS-5407. Fix typos in DFSClientCache (Haohui Mai via brandonli)
+
+ HDFS-5548. Use ConcurrentHashMap in portmap (Haohui Mai via brandonli)
+
+ HDFS-5577. NFS user guide update (brandonli)
+
+ HDFS-5563. NFS gateway should commit the buffered data when read request comes
+ after write to the same file (brandonli)
+
+ HDFS-4997. libhdfs doesn't return correct error codes in most cases (cmccabe)
+
+ HDFS-5587. add debug information when NFS fails to start with duplicate user
+ or group names (brandonli)
+
+ HDFS-5590. Block ID and generation stamp may be reused when persistBlocks is
+ set to false. (jing9)
+
+ HDFS-5353. Short circuit reads fail when dfs.encrypt.data.transfer is
+ enabled. (Colin Patrick McCabe via jing9)
+
+ HDFS-5283. Under construction blocks only inside snapshots should not be
+ counted in safemode threshhold. (Vinay via szetszwo)
+
+ HDFS-5257. addBlock() retry should return LocatedBlock with locations else client
+ will get AIOBE. (Vinay via jing9)
+
+ HDFS-5427. Not able to read deleted files from snapshot directly under
+ snapshottable dir after checkpoint and NN restart. (Vinay via jing9)
+
+ HDFS-5443. Delete 0-sized block when deleting an under-construction file that
+ is included in snapshot. (jing9)
+
+ HDFS-5476. Snapshot: clean the blocks/files/directories under a renamed
+ file/directory while deletion. (jing9)
+
+ HDFS-5425. Renaming underconstruction file with snapshots can make NN failure on
+ restart. (jing9 and Vinay)
+
+ HDFS-5474. Deletesnapshot can make Namenode in safemode on NN restarts.
+ (Sathish via jing9)
+
+ HDFS-5504. In HA mode, OP_DELETE_SNAPSHOT is not decrementing the safemode threshold,
+ leads to NN safemode. (Vinay via jing9)
+
+ HDFS-5428. Under construction files deletion after snapshot+checkpoint+nn restart
+ leads nn safemode. (jing9)
+
+ HDFS-5074. Allow starting up from an fsimage checkpoint in the middle of a
+ segment. (Todd Lipcon via atm)
+
+ HDFS-4201. NPE in BPServiceActor#sendHeartBeat. (jxiang via cmccabe)
+
+ HDFS-5666. Fix inconsistent synchronization in BPOfferService (jxiang via cmccabe)
+
+ HDFS-5657. race condition causes writeback state error in NFS gateway (brandonli)
+
+ HDFS-5661. Browsing FileSystem via web ui, should use datanode's fqdn instead of ip
+ address. (Benoy Antony via jing9)
+
+ HDFS-5582. hdfs getconf -excludeFile or -includeFile always failed (sathish
+ via cmccabe)
+
+ HDFS-5671. Fix socket leak in DFSInputStream#getBlockReader. (JamesLi via umamahesh)
+
+ HDFS-5649. Unregister NFS and Mount service when NFS gateway is shutting down.
+ (brandonli)
+
+ HDFS-5789. Some of snapshot APIs missing checkOperation double check in fsn. (umamahesh)
+
+ HDFS-5343. When cat command is issued on snapshot files getting unexpected result.
+ (Sathish via umamahesh)
+
+ HDFS-5297. Fix dead links in HDFS site documents. (Akira Ajisaka via
+ Arpit Agarwal)
+
+ HDFS-5830. WebHdfsFileSystem.getFileBlockLocations throws
+ IllegalArgumentException when accessing another cluster. (Yongjun Zhang via
+ Colin Patrick McCabe)
+
+ HDFS-5833. Fix SecondaryNameNode javadoc. (Bangtao Zhou via Arpit Agarwal)
+
+ HDFS-5844. Fix broken link in WebHDFS.apt.vm. (Akira Ajisaka via
+ Arpit Agarwal)
+
HDFS-5034. Remove debug prints from GetFileLinkInfo (Andrew Wang via Colin
Patrick McCabe)
@@ -602,6 +833,12 @@ Release 2.4.0 - UNRELEASED
HDFS-5728. Block recovery will fail if the metafile does not have crc
for all chunks of the block (Vinay via kihwal)
+ HDFS-5845. SecondaryNameNode dies when checkpointing with cache pools.
+ (wang)
+
+ HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs
+ on a secure cluster. (jing9)
+
BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS
HDFS-4985. Add storage type to the protocol and expose it in block report
@@ -939,224 +1176,6 @@ Release 2.4.0 - UNRELEASED
HDFS-5724. modifyCacheDirective logging audit log command wrongly as
addCacheDirective (Uma Maheswara Rao G via Colin Patrick McCabe)
-
-Release 2.3.0 - UNRELEASED
-
- INCOMPATIBLE CHANGES
-
- NEW FEATURES
-
- IMPROVEMENTS
-
- HDFS-5360. Improvement of usage message of renameSnapshot and
- deleteSnapshot. (Shinichi Yamashita via wang)
-
- HDFS-5331. make SnapshotDiff.java to a o.a.h.util.Tool interface implementation.
- (Vinayakumar B via umamahesh)
-
- HDFS-4657. Limit the number of blocks logged by the NN after a block
- report to a configurable value. (Aaron T. Myers via Colin Patrick
- McCabe)
-
- HDFS-5344. Make LsSnapshottableDir as Tool interface implementation. (Sathish via umamahesh)
-
- HDFS-5544. Adding Test case For Checking dfs.checksum type as NULL value. (Sathish via umamahesh)
-
- HDFS-5568. Support includeSnapshots option with Fsck command. (Vinayakumar B via umamahesh)
-
- HDFS-4983. Numeric usernames do not work with WebHDFS FS. (Yongjun Zhang via
- jing9)
-
- HDFS-5592. statechangeLog of completeFile should be logged only in case of success.
- (Vinayakumar via umamahesh)
-
- HDFS-5662. Can't decommission a DataNode due to file's replication factor
- larger than the rest of the cluster size. (brandonli)
-
- HDFS-5068. Convert NNThroughputBenchmark to a Tool to allow generic options.
- (shv)
-
- HDFS-5675. Add Mkdirs operation to NNThroughputBenchmark.
- (Plamen Jeliazkov via shv)
-
- HDFS-5677. Need error checking for HA cluster configuration.
- (Vincent Sheffer via cos)
-
- HDFS-5825. Use FileUtils.copyFile() to implement DFSTestUtils.copyFile().
- (Haohui Mai via Arpit Agarwal)
-
- OPTIMIZATIONS
-
- BUG FIXES
-
- HDFS-5307. Support both HTTP and HTTPS in jsp pages (Haohui Mai via
- brandonli)
-
- HDFS-5291. Standby namenode after transition to active goes into safemode.
- (jing9)
-
- HDFS-5317. Go back to DFS Home link does not work on datanode webUI
- (Haohui Mai via brandonli)
-
- HDFS-5316. Namenode ignores the default https port (Haohui Mai via
- brandonli)
-
- HDFS-5281. COMMIT request should not block. (brandonli)
-
- HDFS-5337. should do hsync for a commit request even there is no pending
- writes (brandonli)
-
- HDFS-5335. Hive query failed with possible race in dfs output stream.
- (Haohui Mai via suresh)
-
- HDFS-5322. HDFS delegation token not found in cache errors seen on secure HA
- clusters. (jing9)
-
- HDFS-5329. Update FSNamesystem#getListing() to handle inode path in startAfter
- token. (brandonli)
-
- HDFS-5330. fix readdir and readdirplus for large directories (brandonli)
-
- HDFS-5370. Typo in Error Message: different between range in condition
- and range in error message. (Kousuke Saruta via suresh)
-
- HDFS-5365. Fix libhdfs compile error on FreeBSD9. (Radim Kolar via cnauroth)
-
- HDFS-5347. Add HDFS NFS user guide. (brandonli)
-
- HDFS-5403. WebHdfs client cannot communicate with older WebHdfs servers
- post HDFS-5306. (atm)
-
- HDFS-5171. NFS should create input stream for a file and try to share it
- with multiple read requests. (Haohui Mai via brandonli)
-
- HDFS-5413. hdfs.cmd does not support passthrough to any arbitrary class.
- (cnauroth)
-
- HDFS-5433. When reloading fsimage during checkpointing, we should clear
- existing snapshottable directories. (Aaron T. Myers via wang)
-
- HDFS-5432. TestDatanodeJsp fails on Windows due to assumption that loopback
- address resolves to host name localhost. (cnauroth)
-
- HDFS-5065. TestSymlinkHdfsDisable fails on Windows. (ivanmi)
-
- HDFS-4633 TestDFSClientExcludedNodes fails sporadically if excluded nodes
- cache expires too quickly (Chris Nauroth via Sanjay)
-
- HDFS-5037. Active NN should trigger its own edit log rolls (wang)
-
- HDFS-5035. getFileLinkStatus and rename do not correctly check permissions
- of symlinks. (Andrew Wang via Colin Patrick McCabe)
-
- HDFS-5456. NameNode startup progress creates new steps if caller attempts to
- create a counter for a step that doesn't already exist. (cnauroth)
-
- HDFS-5458. Datanode failed volume threshold ignored if exception is thrown
- in getDataDirsFromURIs. (Mike Mellenthin via wang)
-
- HDFS-5252. Stable write is not handled correctly in someplace. (brandonli)
-
- HDFS-5364. Add OpenFileCtx cache. (brandonli)
-
- HDFS-5469. Add configuration property for the sub-directroy export path
- (brandonli)
-
- HDFS-5519. COMMIT handler should update the commit status after sync
- (brandonli)
-
- HDFS-5372. In FSNamesystem, hasReadLock() returns false if the current thread
- holds the write lock (VinayaKumar B via umamahesh)
-
- HDFS-4516. Client crash after block allocation and NN switch before lease recovery for
- the same file can cause readers to fail forever (VinaayKumar B via umamahesh)
-
- HDFS-5014. Process register commands with out holding BPOfferService lock.
- (Vinaykumar B via umamahesh)
-
- HDFS-5288. Close idle connections in portmap (Haohui Mai via brandonli)
-
- HDFS-5407. Fix typos in DFSClientCache (Haohui Mai via brandonli)
-
- HDFS-5548. Use ConcurrentHashMap in portmap (Haohui Mai via brandonli)
-
- HDFS-5577. NFS user guide update (brandonli)
-
- HDFS-5563. NFS gateway should commit the buffered data when read request comes
- after write to the same file (brandonli)
-
- HDFS-4997. libhdfs doesn't return correct error codes in most cases (cmccabe)
-
- HDFS-5587. add debug information when NFS fails to start with duplicate user
- or group names (brandonli)
-
- HDFS-5590. Block ID and generation stamp may be reused when persistBlocks is
- set to false. (jing9)
-
- HDFS-5353. Short circuit reads fail when dfs.encrypt.data.transfer is
- enabled. (Colin Patrick McCabe via jing9)
-
- HDFS-5283. Under construction blocks only inside snapshots should not be
- counted in safemode threshhold. (Vinay via szetszwo)
-
- HDFS-5257. addBlock() retry should return LocatedBlock with locations else client
- will get AIOBE. (Vinay via jing9)
-
- HDFS-5427. Not able to read deleted files from snapshot directly under
- snapshottable dir after checkpoint and NN restart. (Vinay via jing9)
-
- HDFS-5443. Delete 0-sized block when deleting an under-construction file that
- is included in snapshot. (jing9)
-
- HDFS-5476. Snapshot: clean the blocks/files/directories under a renamed
- file/directory while deletion. (jing9)
-
- HDFS-5425. Renaming underconstruction file with snapshots can make NN failure on
- restart. (jing9 and Vinay)
-
- HDFS-5474. Deletesnapshot can make Namenode in safemode on NN restarts.
- (Sathish via jing9)
-
- HDFS-5504. In HA mode, OP_DELETE_SNAPSHOT is not decrementing the safemode threshold,
- leads to NN safemode. (Vinay via jing9)
-
- HDFS-5428. Under construction files deletion after snapshot+checkpoint+nn restart
- leads nn safemode. (jing9)
-
- HDFS-5074. Allow starting up from an fsimage checkpoint in the middle of a
- segment. (Todd Lipcon via atm)
-
- HDFS-4201. NPE in BPServiceActor#sendHeartBeat. (jxiang via cmccabe)
-
- HDFS-5666. Fix inconsistent synchronization in BPOfferService (jxiang via cmccabe)
-
- HDFS-5657. race condition causes writeback state error in NFS gateway (brandonli)
-
- HDFS-5661. Browsing FileSystem via web ui, should use datanode's fqdn instead of ip
- address. (Benoy Antony via jing9)
-
- HDFS-5582. hdfs getconf -excludeFile or -includeFile always failed (sathish
- via cmccabe)
-
- HDFS-5671. Fix socket leak in DFSInputStream#getBlockReader. (JamesLi via umamahesh)
-
- HDFS-5649. Unregister NFS and Mount service when NFS gateway is shutting down.
- (brandonli)
-
- HDFS-5789. Some of snapshot APIs missing checkOperation double check in fsn. (umamahesh)
-
- HDFS-5343. When cat command is issued on snapshot files getting unexpected result.
- (Sathish via umamahesh)
-
- HDFS-5297. Fix dead links in HDFS site documents. (Akira Ajisaka via
- Arpit Agarwal)
-
- HDFS-5830. WebHdfsFileSystem.getFileBlockLocations throws
- IllegalArgumentException when accessing another cluster. (Yongjun Zhang via
- Colin Patrick McCabe)
-
- HDFS-5833. Fix SecondaryNameNode javadoc. (Bangtao Zhou via Arpit Agarwal)
-
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1561944-1563041
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Fri Jan 31 03:24:24 2014
@@ -84,7 +84,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
@@ -1539,7 +1539,7 @@ public class DFSUtil {
return policy;
}
- public static HttpServer.Builder loadSslConfToHttpServerBuilder(HttpServer.Builder builder,
+ public static HttpServer2.Builder loadSslConfToHttpServerBuilder(HttpServer2.Builder builder,
Configuration sslConf) {
return builder
.needsClientAuth(
@@ -1644,13 +1644,13 @@ public class DFSUtil {
* namenode can use to initialize their HTTP / HTTPS server.
*
*/
- public static HttpServer.Builder httpServerTemplateForNNAndJN(
+ public static HttpServer2.Builder httpServerTemplateForNNAndJN(
Configuration conf, final InetSocketAddress httpAddr,
final InetSocketAddress httpsAddr, String name, String spnegoUserNameKey,
String spnegoKeytabFileKey) throws IOException {
HttpConfig.Policy policy = getHttpPolicy(conf);
- HttpServer.Builder builder = new HttpServer.Builder().setName(name)
+ HttpServer2.Builder builder = new HttpServer2.Builder().setName(name)
.setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
.setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
.setUsernameConfKey(spnegoUserNameKey)
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeHttpServer.java Fri Jan 31 03:24:24 2014
@@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
/**
@@ -38,7 +38,7 @@ import org.apache.hadoop.net.NetUtils;
public class JournalNodeHttpServer {
public static final String JN_ATTRIBUTE_KEY = "localjournal";
- private HttpServer httpServer;
+ private HttpServer2 httpServer;
private JournalNode localJournalNode;
private final Configuration conf;
@@ -56,7 +56,7 @@ public class JournalNodeHttpServer {
DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_DEFAULT);
InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
- HttpServer.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
+ HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
httpAddr, httpsAddr, "journal",
DFSConfigKeys.DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY,
DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY);
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri Jan 31 03:24:24 2014
@@ -120,7 +120,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -235,7 +235,7 @@ public class DataNode extends Configured
private volatile boolean heartbeatsDisabledForTests = false;
private DataStorage storage = null;
- private HttpServer infoServer = null;
+ private HttpServer2 infoServer = null;
private int infoPort;
private int infoSecurePort;
@@ -358,7 +358,7 @@ public class DataNode extends Configured
* Http Policy is decided.
*/
private void startInfoServer(Configuration conf) throws IOException {
- HttpServer.Builder builder = new HttpServer.Builder().setName("datanode")
+ HttpServer2.Builder builder = new HttpServer2.Builder().setName("datanode")
.setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")));
HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Fri Jan 31 03:24:24 2014
@@ -655,8 +655,9 @@ class DataXceiver extends Receiver imple
final BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn);
final DataChecksum checksum = header.getChecksum();
final int bytesPerCRC = checksum.getBytesPerChecksum();
- final long crcPerBlock = (metadataIn.getLength()
- - BlockMetadataHeader.getHeaderSize())/checksum.getChecksumSize();
+ final long crcPerBlock = checksum.getChecksumSize() > 0
+ ? (metadataIn.getLength() - BlockMetadataHeader.getHeaderSize())/checksum.getChecksumSize()
+ : 0;
//compute block checksum
final MD5Hash md5 = MD5Hash.digest(checksumIn);
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java Fri Jan 31 03:24:24 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
import org.mortbay.jetty.Connector;
@@ -119,7 +119,7 @@ public class SecureDataNodeStarter imple
// certificates if they are communicating through SSL.
Connector listener = null;
if (policy.isHttpEnabled()) {
- listener = HttpServer.createDefaultChannelConnector();
+ listener = HttpServer2.createDefaultChannelConnector();
InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf);
listener.setHost(infoSocAddr.getHostName());
listener.setPort(infoSocAddr.getPort());
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java Fri Jan 31 03:24:24 2014
@@ -195,6 +195,17 @@ public final class CacheManager {
}
+ /**
+ * Resets all tracked directives and pools. Called during 2NN checkpointing to
+ * reset FSNamesystem state. See {FSNamesystem{@link #clear()}.
+ */
+ void clear() {
+ directivesById.clear();
+ directivesByPath.clear();
+ cachePools.clear();
+ nextDirectiveId = 1;
+ }
+
public void startMonitorThread() {
crmLock.lock();
try {
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Jan 31 03:24:24 2014
@@ -549,6 +549,7 @@ public class FSNamesystem implements Nam
leaseManager.removeAllLeases();
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
snapshotManager.clearSnapshottableDirs();
+ cacheManager.clear();
}
@VisibleForTesting
@@ -3817,7 +3818,7 @@ public class FSNamesystem implements Nam
final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
if (diff > 0) {
try {
- String path = leaseManager.findPath(fileINode);
+ String path = fileINode.getFullPathName();
dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication());
} catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e);
@@ -4019,7 +4020,7 @@ public class FSNamesystem implements Nam
@VisibleForTesting
String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
throws IOException {
- String src = leaseManager.findPath(pendingFile);
+ String src = pendingFile.getFullPathName();
// commit the last block and complete it if it has minimum replicas
commitOrCompleteLastBlock(pendingFile, storedBlock);
@@ -4041,7 +4042,7 @@ public class FSNamesystem implements Nam
@VisibleForTesting
String persistBlocks(INodeFile pendingFile, boolean logRetryCache)
throws IOException {
- String src = leaseManager.findPath(pendingFile);
+ String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache);
return src;
}
@@ -5940,7 +5941,7 @@ public class FSNamesystem implements Nam
.getDatanodeStorageInfos(newNodes, newStorageIDs);
blockinfo.setExpectedLocations(storages);
- String src = leaseManager.findPath(pendingFile);
+ String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java Fri Jan 31 03:24:24 2014
@@ -47,7 +47,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.security.UserGroupInformation;
@@ -287,7 +287,7 @@ public class GetImageServlet extends Htt
}
}
- if (HttpServer.userHasAdministratorAccess(context, remoteUser)) {
+ if (HttpServer2.userHasAdministratorAccess(context, remoteUser)) {
LOG.info("GetImageServlet allowing administrator: " + remoteUser);
return true;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Fri Jan 31 03:24:24 2014
@@ -180,24 +180,6 @@ public class LeaseManager {
}
/**
- * Finds the pathname for the specified pendingFile
- */
- public synchronized String findPath(INodeFile pendingFile)
- throws IOException {
- FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
- Preconditions.checkArgument(uc != null);
- Lease lease = getLease(uc.getClientName());
- if (lease != null) {
- String src = lease.findPath(pendingFile);
- if (src != null) {
- return src;
- }
- }
- throw new IOException("pendingFile (=" + pendingFile + ") not found."
- + "(lease=" + lease + ")");
- }
-
- /**
* Renew the lease(s) held by the given client
*/
synchronized void renewLease(String holder) {
@@ -252,24 +234,6 @@ public class LeaseManager {
return now() - lastUpdate > softLimit;
}
- /**
- * @return the path associated with the pendingFile and null if not found.
- */
- private String findPath(INodeFile pendingFile) {
- try {
- for (String src : paths) {
- INode node = fsnamesystem.dir.getINode(src);
- if (node == pendingFile
- || (node.isFile() && node.asFile() == pendingFile)) {
- return src;
- }
- }
- } catch (UnresolvedLinkException e) {
- throw new AssertionError("Lease files should reside on this FS");
- }
- return null;
- }
-
/** Does this lease contain any path? */
boolean hasPath() {return !paths.isEmpty();}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Fri Jan 31 03:24:24 2014
@@ -37,7 +37,7 @@ import org.apache.hadoop.hdfs.web.WebHdf
import org.apache.hadoop.hdfs.web.resources.Param;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -47,7 +47,7 @@ import org.apache.hadoop.security.UserGr
*/
@InterfaceAudience.Private
public class NameNodeHttpServer {
- private HttpServer httpServer;
+ private HttpServer2 httpServer;
private final Configuration conf;
private final NameNode nn;
@@ -68,7 +68,7 @@ public class NameNodeHttpServer {
}
private void initWebHdfs(Configuration conf) throws IOException {
- if (WebHdfsFileSystem.isEnabled(conf, HttpServer.LOG)) {
+ if (WebHdfsFileSystem.isEnabled(conf, HttpServer2.LOG)) {
// set user pattern based on configuration file
UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
@@ -77,9 +77,9 @@ public class NameNodeHttpServer {
final String classname = AuthFilter.class.getName();
final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
Map<String, String> params = getAuthFilterParams(conf);
- HttpServer.defineFilter(httpServer.getWebAppContext(), name, classname, params,
+ HttpServer2.defineFilter(httpServer.getWebAppContext(), name, classname, params,
new String[]{pathSpec});
- HttpServer.LOG.info("Added filter '" + name + "' (class=" + classname + ")");
+ HttpServer2.LOG.info("Added filter '" + name + "' (class=" + classname + ")");
// add webhdfs packages
httpServer.addJerseyResourcePackage(
@@ -103,7 +103,7 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT);
InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
- HttpServer.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
+ HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
httpAddr, httpsAddr, "hdfs",
DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY,
DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY);
@@ -152,7 +152,7 @@ public class NameNodeHttpServer {
SecurityUtil.getServerPrincipal(principalInConf,
bindAddress.getHostName()));
} else if (UserGroupInformation.isSecurityEnabled()) {
- HttpServer.LOG.error(
+ HttpServer2.LOG.error(
"WebHDFS and security are enabled, but configuration property '" +
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY +
"' is not set.");
@@ -164,7 +164,7 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY,
httpKeytab);
} else if (UserGroupInformation.isSecurityEnabled()) {
- HttpServer.LOG.error(
+ HttpServer2.LOG.error(
"WebHDFS and security are enabled, but configuration property '" +
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY +
"' is not set.");
@@ -214,7 +214,7 @@ public class NameNodeHttpServer {
httpServer.setAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY, prog);
}
- private static void setupServlets(HttpServer httpServer, Configuration conf) {
+ private static void setupServlets(HttpServer2 httpServer, Configuration conf) {
httpServer.addInternalServlet("startupProgress",
StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class);
httpServer.addInternalServlet("getDelegationToken",
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Fri Jan 31 03:24:24 2014
@@ -65,7 +65,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -113,7 +113,7 @@ public class SecondaryNameNode implement
private Configuration conf;
private InetSocketAddress nameNodeAddr;
private volatile boolean shouldRun;
- private HttpServer infoServer;
+ private HttpServer2 infoServer;
private URL imageListenURL;
private Collection<URI> checkpointDirs;
@@ -257,7 +257,7 @@ public class SecondaryNameNode implement
DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_DEFAULT);
InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
- HttpServer.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
+ HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
httpAddr, httpsAddr, "secondary",
DFSConfigKeys.DFS_SECONDARY_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY,
DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY);
@@ -1003,7 +1003,12 @@ public class SecondaryNameNode implement
sig.mostRecentCheckpointTxId + " even though it should have " +
"just been downloaded");
}
- dstImage.reloadFromImageFile(file, dstNamesystem);
+ dstNamesystem.writeLock();
+ try {
+ dstImage.reloadFromImageFile(file, dstNamesystem);
+ } finally {
+ dstNamesystem.writeUnlock();
+ }
dstNamesystem.dir.imageLoadComplete();
}
// error simulation code for junit test
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java Fri Jan 31 03:24:24 2014
@@ -620,7 +620,7 @@ public class CacheAdmin extends Configur
"directives being added to the pool. This can be specified in " +
"seconds, minutes, hours, and days, e.g. 120s, 30m, 4h, 2d. " +
"Valid units are [smhd]. By default, no maximum is set. " +
- "This can also be manually specified by \"never\".");
+ "A value of \"never\" specifies that there is no limit.");
return getShortUsage() + "\n" +
"Add a new cache pool.\n\n" +
listing.toString();
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Fri Jan 31 03:24:24 2014
@@ -185,8 +185,8 @@ public class DelegationTokenFetcher {
} else {
// otherwise we are fetching
if (webUrl != null) {
- Credentials creds = getDTfromRemote(connectionFactory, new URI(webUrl),
- renewer);
+ Credentials creds = getDTfromRemote(connectionFactory, new URI(
+ webUrl), renewer, null);
creds.writeTokenStorageFile(tokenFile, conf);
for (Token<?> token : creds.getAllTokens()) {
if(LOG.isDebugEnabled()) {
@@ -213,12 +213,17 @@ public class DelegationTokenFetcher {
}
static public Credentials getDTfromRemote(URLConnectionFactory factory,
- URI nnUri, String renewer) throws IOException {
+ URI nnUri, String renewer, String proxyUser) throws IOException {
StringBuilder buf = new StringBuilder(nnUri.toString())
.append(GetDelegationTokenServlet.PATH_SPEC);
+ String separator = "?";
if (renewer != null) {
buf.append("?").append(GetDelegationTokenServlet.RENEWER).append("=")
.append(renewer);
+ separator = "&";
+ }
+ if (proxyUser != null) {
+ buf.append(separator).append("doas=").append(proxyUser);
}
boolean isHttps = nnUri.getScheme().equals("https");
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java Fri Jan 31 03:24:24 2014
@@ -57,7 +57,6 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Progressable;
@@ -234,17 +233,23 @@ public class HftpFileSystem extends File
}
@Override
- public synchronized Token<?> getDelegationToken(final String renewer
- ) throws IOException {
+ public synchronized Token<?> getDelegationToken(final String renewer)
+ throws IOException {
try {
- //Renew TGT if needed
- ugi.checkTGTAndReloginFromKeytab();
- return ugi.doAs(new PrivilegedExceptionAction<Token<?>>() {
+ // Renew TGT if needed
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ final String proxyUser = connectUgi == null ? null : ugi
+ .getShortUserName();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
+ return connectUgi.doAs(new PrivilegedExceptionAction<Token<?>>() {
@Override
public Token<?> run() throws IOException {
Credentials c;
try {
- c = DelegationTokenFetcher.getDTfromRemote(connectionFactory, nnUri, renewer);
+ c = DelegationTokenFetcher.getDTfromRemote(connectionFactory,
+ nnUri, renewer, proxyUser);
} catch (IOException e) {
if (e.getCause() instanceof ConnectException) {
LOG.warn("Couldn't connect to " + nnUri +
@@ -299,13 +304,13 @@ public class HftpFileSystem extends File
* @return user_shortname,group1,group2...
*/
private String getEncodedUgiParameter() {
- StringBuilder ugiParamenter = new StringBuilder(
+ StringBuilder ugiParameter = new StringBuilder(
ServletUtil.encodeQueryValue(ugi.getShortUserName()));
for(String g: ugi.getGroupNames()) {
- ugiParamenter.append(",");
- ugiParamenter.append(ServletUtil.encodeQueryValue(g));
+ ugiParameter.append(",");
+ ugiParameter.append(ServletUtil.encodeQueryValue(g));
}
- return ugiParamenter.toString();
+ return ugiParameter.toString();
}
/**
@@ -675,30 +680,48 @@ public class HftpFileSystem extends File
@SuppressWarnings("unchecked")
@Override
- public long renewDelegationToken(Token<?> token) throws IOException {
+ public long renewDelegationToken(final Token<?> token) throws IOException {
// update the kerberos credentials, if they are coming from a keytab
- UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
- InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
try {
- return DelegationTokenFetcher.renewDelegationToken(connectionFactory,
- DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
- (Token<DelegationTokenIdentifier>) token);
- } catch (AuthenticationException e) {
+ return connectUgi.doAs(new PrivilegedExceptionAction<Long>() {
+ @Override
+ public Long run() throws Exception {
+ InetSocketAddress serviceAddr = SecurityUtil
+ .getTokenServiceAddr(token);
+ return DelegationTokenFetcher.renewDelegationToken(connectionFactory,
+ DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
+ (Token<DelegationTokenIdentifier>) token);
+ }
+ });
+ } catch (InterruptedException e) {
throw new IOException(e);
}
}
@SuppressWarnings("unchecked")
@Override
- public void cancelDelegationToken(Token<?> token) throws IOException {
- // update the kerberos credentials, if they are coming from a keytab
- UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
- InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
+ public void cancelDelegationToken(final Token<?> token) throws IOException {
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
try {
- DelegationTokenFetcher.cancelDelegationToken(connectionFactory, DFSUtil
- .createUri(getUnderlyingProtocol(), serviceAddr),
- (Token<DelegationTokenIdentifier>) token);
- } catch (AuthenticationException e) {
+ connectUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ InetSocketAddress serviceAddr = SecurityUtil
+ .getTokenServiceAddr(token);
+ DelegationTokenFetcher.cancelDelegationToken(connectionFactory,
+ DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
+ (Token<DelegationTokenIdentifier>) token);
+ return null;
+ }
+ });
+ } catch (InterruptedException e) {
throw new IOException(e);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm Fri Jan 31 03:24:24 2014
@@ -22,110 +22,140 @@ Centralized Cache Management in HDFS
%{toc|section=1|fromDepth=2|toDepth=4}
-* {Background}
+* {Overview}
- Normally, HDFS relies on the operating system to cache data it reads from disk.
- However, HDFS can also be configured to use centralized cache management. Under
- centralized cache management, the HDFS NameNode itself decides which blocks
- should be cached, and where they should be cached.
-
- Centralized cache management has several advantages. First of all, it
- prevents frequently used block files from being evicted from memory. This is
- particularly important when the size of the working set exceeds the size of
- main memory, which is true for many big data applications. Secondly, when
- HDFS decides what should be cached, it can let clients know about this
- information through the getFileBlockLocations API. Finally, when the DataNode
- knows a block is locked into memory, it can provide access to that block via
- mmap.
+ <Centralized cache management> in HDFS is an explicit caching mechanism that
+ allows users to specify <paths> to be cached by HDFS. The NameNode will
+ communicate with DataNodes that have the desired blocks on disk, and instruct
+ them to cache the blocks in off-heap caches.
+
+ Centralized cache management in HDFS has many significant advantages.
+
+ [[1]] Explicit pinning prevents frequently used data from being evicted from
+ memory. This is particularly important when the size of the working set
+ exceeds the size of main memory, which is common for many HDFS workloads.
+
+ [[1]] Because DataNode caches are managed by the NameNode, applications can
+ query the set of cached block locations when making task placement decisions.
+ Co-locating a task with a cached block replica improves read performance.
+
+ [[1]] When block has been cached by a DataNode, clients can use a new ,
+ more-efficient, zero-copy read API. Since checksum verification of cached
+ data is done once by the DataNode, clients can incur essentially zero
+ overhead when using this new API.
+
+ [[1]] Centralized caching can improve overall cluster memory utilization.
+ When relying on the OS buffer cache at each DataNode, repeated reads of
+ a block will result in all <n> replicas of the block being pulled into
+ buffer cache. With centralized cache management, a user can explicitly pin
+ only <m> of the <n> replicas, saving <n-m> memory.
* {Use Cases}
- Centralized cache management is most useful for files which are accessed very
- often. For example, a "fact table" in Hive which is often used in joins is a
- good candidate for caching. On the other hand, when running a classic
- "word count" MapReduce job which counts the number of words in each
- document, there may not be any good candidates for caching, since all the
- files may be accessed exactly once.
+ Centralized cache management is useful for files that accessed repeatedly.
+ For example, a small <fact table> in Hive which is often used for joins is a
+ good candidate for caching. On the other hand, caching the input of a <
+ one year reporting query> is probably less useful, since the
+ historical data might only be read once.
+
+ Centralized cache management is also useful for mixed workloads with
+ performance SLAs. Caching the working set of a high-priority workload
+ insures that it does not contend for disk I/O with a low-priority workload.
* {Architecture}
[images/caching.png] Caching Architecture
- With centralized cache management, the NameNode coordinates all caching
- across the cluster. It receives cache information from each DataNode via the
- cache report, a periodic message that describes all the blocks IDs cached on
- a given DataNode. The NameNode will reply to DataNode heartbeat messages
- with commands telling it which blocks to cache and which to uncache.
-
- The NameNode stores a set of path cache directives, which tell it which files
- to cache. The NameNode also stores a set of cache pools, which are groups of
- cache directives. These directives and pools are persisted to the edit log
- and fsimage, and will be loaded if the cluster is restarted.
-
- Periodically, the NameNode rescans the namespace, to see which blocks need to
- be cached based on the current set of path cache directives. Rescans are also
- triggered by relevant user actions, such as adding or removing a cache
- directive or removing a cache pool.
-
- Cache directives also may specific a numeric cache replication, which is the
- number of replicas to cache. This number may be equal to or smaller than the
- file's block replication. If multiple cache directives cover the same file
- with different cache replication settings, then the highest cache replication
- setting is applied.
+ In this architecture, the NameNode is responsible for coordinating all the
+ DataNode off-heap caches in the cluster. The NameNode periodically receives
+ a <cache report> from each DataNode which describes all the blocks cached
+ on a given DN. The NameNode manages DataNode caches by piggybacking cache and
+ uncache commands on the DataNode heartbeat.
+
+ The NameNode queries its set of <cache directives> to determine
+ which paths should be cached. Cache directives are persistently stored in the
+ fsimage and edit log, and can be added, removed, and modified via Java and
+ command-line APIs. The NameNode also stores a set of <cache pools>,
+ which are administrative entities used to group cache directives together for
+ resource management and enforcing permissions.
+
+ The NameNode periodically rescans the namespace and active cache directives
+ to determine which blocks need to be cached or uncached and assign caching
+ work to DataNodes. Rescans can also be triggered by user actions like adding
+ or removing a cache directive or removing a cache pool.
We do not currently cache blocks which are under construction, corrupt, or
otherwise incomplete. If a cache directive covers a symlink, the symlink
target is not cached.
- Caching is currently done on a per-file basis, although we would like to add
- block-level granularity in the future.
+ Caching is currently done on the file or directory-level. Block and sub-block
+ caching is an item of future work.
-* {Interface}
+* {Concepts}
- The NameNode stores a list of "cache directives." These directives contain a
- path as well as the number of times blocks in that path should be replicated.
+** {Cache directive}
- Paths can be either directories or files. If the path specifies a file, that
- file is cached. If the path specifies a directory, all the files in the
- directory will be cached. However, this process is not recursive-- only the
- direct children of the directory will be cached.
-
-** {hdfs cacheadmin Shell}
-
- Path cache directives can be created by the <<<hdfs cacheadmin
- -addDirective>>> command and removed via the <<<hdfs cacheadmin
- -removeDirective>>> command. To list the current path cache directives, use
- <<<hdfs cacheadmin -listDirectives>>>. Each path cache directive has a
- unique 64-bit ID number which will not be reused if it is deleted. To remove
- all path cache directives with a specified path, use <<<hdfs cacheadmin
- -removeDirectives>>>.
-
- Directives are grouped into "cache pools." Each cache pool gets a share of
- the cluster's resources. Additionally, cache pools are used for
- authentication. Cache pools have a mode, user, and group, similar to regular
- files. The same authentication rules are applied as for normal files. So, for
- example, if the mode is 0777, any user can add or remove directives from the
- cache pool. If the mode is 0644, only the owner can write to the cache pool,
- but anyone can read from it. And so forth.
-
- Cache pools are identified by name. They can be created by the <<<hdfs
- cacheAdmin -addPool>>> command, modified by the <<<hdfs cacheadmin
- -modifyPool>>> command, and removed via the <<<hdfs cacheadmin
- -removePool>>> command. To list the current cache pools, use <<<hdfs
- cacheAdmin -listPools>>>
+ A <cache directive> defines a path that should be cached. Paths can be either
+ directories or files. Directories are cached non-recursively, meaning only
+ files in the first-level listing of the directory.
+
+ Directives also specify additional parameters, such as the cache replication
+ factor and expiration time. The replication factor specifies the number of
+ block replicas to cache. If multiple cache directives refer to the same file,
+ the maximum cache replication factor is applied.
+
+ The expiration time is specified on the command line as a <time-to-live
+ (TTL)>, a relative expiration time in the future. After a cache directive
+ expires, it is no longer considered by the NameNode when making caching
+ decisions.
+
+** {Cache pool}
+
+ A <cache pool> is an administrative entity used to manage groups of cache
+ directives. Cache pools have UNIX-like <permissions>, which restrict which
+ users and groups have access to the pool. Write permissions allow users to
+ add and remove cache directives to the pool. Read permissions allow users to
+ list the cache directives in a pool, as well as additional metadata. Execute
+ permissions are unused.
+
+ Cache pools are also used for resource management. Pools can enforce a
+ maximum <limit>, which restricts the number of bytes that can be cached in
+ aggregate by directives in the pool. Normally, the sum of the pool limits
+ will approximately equal the amount of aggregate memory reserved for
+ HDFS caching on the cluster. Cache pools also track a number of statistics
+ to help cluster users determine what is and should be cached.
+
+ Pools also can enforce a maximum time-to-live. This restricts the maximum
+ expiration time of directives being added to the pool.
+
+* {<<<cacheadmin>>> command-line interface}
+
+ On the command-line, administrators and users can interact with cache pools
+ and directives via the <<<hdfs cacheadmin>>> subcommand.
+
+ Cache directives are identified by a unique, non-repeating 64-bit integer ID.
+ IDs will not be reused even if a cache directive is later removed.
+
+ Cache pools are identified by a unique string name.
+
+** {Cache directive commands}
*** {addDirective}
- Usage: <<<hdfs cacheadmin -addDirective -path <path> -replication <replication> -pool <pool-name> >>>
+ Usage: <<<hdfs cacheadmin -addDirective -path <path> -pool <pool-name> [-force] [-replication <replication>] [-ttl <time-to-live>]>>>
Add a new cache directive.
*--+--+
\<path\> | A path to cache. The path can be a directory or a file.
*--+--+
+\<pool-name\> | The pool to which the directive will be added. You must have write permission on the cache pool in order to add new directives.
+*--+--+
+-force | Skips checking of cache pool resource limits.
+*--+--+
\<replication\> | The cache replication factor to use. Defaults to 1.
*--+--+
-\<pool-name\> | The pool to which the directive will be added. You must have write permission on the cache pool in order to add new directives.
+\<time-to-live\> | How long the directive is valid. Can be specified in minutes, hours, and days, e.g. 30m, 4h, 2d. Valid units are [smhd]. "never" indicates a directive that never expires. If unspecified, the directive never expires.
*--+--+
*** {removeDirective}
@@ -150,7 +180,7 @@ Centralized Cache Management in HDFS
*** {listDirectives}
- Usage: <<<hdfs cacheadmin -listDirectives [-path <path>] [-pool <pool>] >>>
+ Usage: <<<hdfs cacheadmin -listDirectives [-stats] [-path <path>] [-pool <pool>]>>>
List cache directives.
@@ -159,10 +189,14 @@ Centralized Cache Management in HDFS
*--+--+
\<pool\> | List only path cache directives in that pool.
*--+--+
+-stats | List path-based cache directive statistics.
+*--+--+
+
+** {Cache pool commands}
*** {addPool}
- Usage: <<<hdfs cacheadmin -addPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-weight <weight>] >>>
+ Usage: <<<hdfs cacheadmin -addPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-limit <limit>] [-maxTtl <maxTtl>>>>
Add a new cache pool.
@@ -175,12 +209,14 @@ Centralized Cache Management in HDFS
*--+--+
\<mode\> | UNIX-style permissions for the pool. Permissions are specified in octal, e.g. 0755. By default, this is set to 0755.
*--+--+
-\<weight\> | Weight of the pool. This is a relative measure of the importance of the pool used during cache resource management. By default, it is set to 100.
+\<limit\> | The maximum number of bytes that can be cached by directives in this pool, in aggregate. By default, no limit is set.
+*--+--+
+\<maxTtl\> | The maximum allowed time-to-live for directives being added to the pool. This can be specified in seconds, minutes, hours, and days, e.g. 120s, 30m, 4h, 2d. Valid units are [smhd]. By default, no maximum is set. A value of \"never\" specifies that there is no limit.
*--+--+
*** {modifyPool}
- Usage: <<<hdfs cacheadmin -modifyPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-weight <weight>] >>>
+ Usage: <<<hdfs cacheadmin -modifyPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-limit <limit>] [-maxTtl <maxTtl>]>>>
Modifies the metadata of an existing cache pool.
@@ -193,7 +229,9 @@ Centralized Cache Management in HDFS
*--+--+
\<mode\> | Unix-style permissions of the pool in octal.
*--+--+
-\<weight\> | Weight of the pool.
+\<limit\> | Maximum number of bytes that can be cached by this pool.
+*--+--+
+\<maxTtl\> | The maximum allowed time-to-live for directives being added to the pool.
*--+--+
*** {removePool}
@@ -208,12 +246,14 @@ Centralized Cache Management in HDFS
*** {listPools}
- Usage: <<<hdfs cacheadmin -listPools [name] >>>
+ Usage: <<<hdfs cacheadmin -listPools [-stats] [<name>]>>>
Display information about one or more cache pools, e.g. name, owner, group,
permissions, etc.
*--+--+
+-stats | Display additional cache pool statistics.
+*--+--+
\<name\> | If specified, list only the named cache pool.
*--+--+
@@ -244,10 +284,12 @@ Centralized Cache Management in HDFS
* dfs.datanode.max.locked.memory
- The DataNode will treat this as the maximum amount of memory it can use for
- its cache. When setting this value, please remember that you will need space
- in memory for other things, such as the Java virtual machine (JVM) itself
- and the operating system's page cache.
+ This determines the maximum amount of memory a DataNode will use for caching.
+ The "locked-in-memory size" ulimit (<<<ulimit -l>>>) of the DataNode user
+ also needs to be increased to match this parameter (see below section on
+ {{OS Limits}}). When setting this value, please remember that you will need
+ space in memory for other things as well, such as the DataNode and
+ application JVM heaps and the operating system page cache.
*** Optional
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm Fri Jan 31 03:24:24 2014
@@ -17,11 +17,11 @@
---
${maven.build.timestamp}
-%{toc|section=1|fromDepth=0}
-
HDFS Architecture
-Introduction
+%{toc|section=1|fromDepth=0}
+
+* Introduction
The Hadoop Distributed File System (HDFS) is a distributed file system
designed to run on commodity hardware. It has many similarities with
@@ -35,9 +35,9 @@ Introduction
is part of the Apache Hadoop Core project. The project URL is
{{http://hadoop.apache.org/}}.
-Assumptions and Goals
+* Assumptions and Goals
-Hardware Failure
+** Hardware Failure
Hardware failure is the norm rather than the exception. An HDFS
instance may consist of hundreds or thousands of server machines, each
@@ -47,7 +47,7 @@ Hardware Failure
non-functional. Therefore, detection of faults and quick, automatic
recovery from them is a core architectural goal of HDFS.
-Streaming Data Access
+** Streaming Data Access
Applications that run on HDFS need streaming access to their data sets.
They are not general purpose applications that typically run on general
@@ -58,7 +58,7 @@ Streaming Data Access
targeted for HDFS. POSIX semantics in a few key areas has been traded
to increase data throughput rates.
-Large Data Sets
+** Large Data Sets
Applications that run on HDFS have large data sets. A typical file in
HDFS is gigabytes to terabytes in size. Thus, HDFS is tuned to support
@@ -66,7 +66,7 @@ Large Data Sets
to hundreds of nodes in a single cluster. It should support tens of
millions of files in a single instance.
-Simple Coherency Model
+** Simple Coherency Model
HDFS applications need a write-once-read-many access model for files. A
file once created, written, and closed need not be changed. This
@@ -75,7 +75,7 @@ Simple Coherency Model
perfectly with this model. There is a plan to support appending-writes
to files in the future.
-âMoving Computation is Cheaper than Moving Dataâ
+** âMoving Computation is Cheaper than Moving Dataâ
A computation requested by an application is much more efficient if it
is executed near the data it operates on. This is especially true when
@@ -86,13 +86,13 @@ Simple Coherency Model
running. HDFS provides interfaces for applications to move themselves
closer to where the data is located.
-Portability Across Heterogeneous Hardware and Software Platforms
+** Portability Across Heterogeneous Hardware and Software Platforms
HDFS has been designed to be easily portable from one platform to
another. This facilitates widespread adoption of HDFS as a platform of
choice for a large set of applications.
-NameNode and DataNodes
+* NameNode and DataNodes
HDFS has a master/slave architecture. An HDFS cluster consists of a
single NameNode, a master server that manages the file system namespace
@@ -127,7 +127,7 @@ NameNode and DataNodes
repository for all HDFS metadata. The system is designed in such a way
that user data never flows through the NameNode.
-The File System Namespace
+* The File System Namespace
HDFS supports a traditional hierarchical file organization. A user or
an application can create directories and store files inside these
@@ -145,7 +145,7 @@ The File System Namespace
replication factor of that file. This information is stored by the
NameNode.
-Data Replication
+* Data Replication
HDFS is designed to reliably store very large files across machines in
a large cluster. It stores each file as a sequence of blocks; all
@@ -164,7 +164,7 @@ Data Replication
[images/hdfsdatanodes.png] HDFS DataNodes
-Replica Placement: The First Baby Steps
+** Replica Placement: The First Baby Steps
The placement of replicas is critical to HDFS reliability and
performance. Optimizing replica placement distinguishes HDFS from most
@@ -210,7 +210,7 @@ Replica Placement: The First Baby Steps
The current, default replica placement policy described here is a work
in progress.
-Replica Selection
+** Replica Selection
To minimize global bandwidth consumption and read latency, HDFS tries
to satisfy a read request from a replica that is closest to the reader.
@@ -219,7 +219,7 @@ Replica Selection
cluster spans multiple data centers, then a replica that is resident in
the local data center is preferred over any remote replica.
-Safemode
+** Safemode
On startup, the NameNode enters a special state called Safemode.
Replication of data blocks does not occur when the NameNode is in the
@@ -234,7 +234,7 @@ Safemode
blocks (if any) that still have fewer than the specified number of
replicas. The NameNode then replicates these blocks to other DataNodes.
-The Persistence of File System Metadata
+* The Persistence of File System Metadata
The HDFS namespace is stored by the NameNode. The NameNode uses a
transaction log called the EditLog to persistently record every change
@@ -273,7 +273,7 @@ The Persistence of File System Metadata
each of these local files and sends this report to the NameNode: this
is the Blockreport.
-The Communication Protocols
+* The Communication Protocols
All HDFS communication protocols are layered on top of the TCP/IP
protocol. A client establishes a connection to a configurable TCP port
@@ -284,13 +284,13 @@ The Communication Protocols
RPCs. Instead, it only responds to RPC requests issued by DataNodes or
clients.
-Robustness
+* Robustness
The primary objective of HDFS is to store data reliably even in the
presence of failures. The three common types of failures are NameNode
failures, DataNode failures and network partitions.
-Data Disk Failure, Heartbeats and Re-Replication
+** Data Disk Failure, Heartbeats and Re-Replication
Each DataNode sends a Heartbeat message to the NameNode periodically. A
network partition can cause a subset of DataNodes to lose connectivity
@@ -306,7 +306,7 @@ Data Disk Failure, Heartbeats and Re-Rep
corrupted, a hard disk on a DataNode may fail, or the replication
factor of a file may be increased.
-Cluster Rebalancing
+** Cluster Rebalancing
The HDFS architecture is compatible with data rebalancing schemes. A
scheme might automatically move data from one DataNode to another if
@@ -316,7 +316,7 @@ Cluster Rebalancing
cluster. These types of data rebalancing schemes are not yet
implemented.
-Data Integrity
+** Data Integrity
It is possible that a block of data fetched from a DataNode arrives
corrupted. This corruption can occur because of faults in a storage
@@ -330,7 +330,7 @@ Data Integrity
to retrieve that block from another DataNode that has a replica of that
block.
-Metadata Disk Failure
+** Metadata Disk Failure
The FsImage and the EditLog are central data structures of HDFS. A
corruption of these files can cause the HDFS instance to be
@@ -350,16 +350,16 @@ Metadata Disk Failure
Currently, automatic restart and failover of the NameNode software to
another machine is not supported.
-Snapshots
+** Snapshots
Snapshots support storing a copy of data at a particular instant of
time. One usage of the snapshot feature may be to roll back a corrupted
HDFS instance to a previously known good point in time. HDFS does not
currently support snapshots but will in a future release.
-Data Organization
+* Data Organization
-Data Blocks
+** Data Blocks
HDFS is designed to support very large files. Applications that are
compatible with HDFS are those that deal with large data sets. These
@@ -370,7 +370,7 @@ Data Blocks
chunks, and if possible, each chunk will reside on a different
DataNode.
-Staging
+** Staging
A client request to create a file does not reach the NameNode
immediately. In fact, initially the HDFS client caches the file data
@@ -397,7 +397,7 @@ Staging
side caching to improve performance. A POSIX requirement has been
relaxed to achieve higher performance of data uploads.
-Replication Pipelining
+** Replication Pipelining
When a client is writing data to an HDFS file, its data is first
written to a local file as explained in the previous section. Suppose
@@ -406,7 +406,7 @@ Replication Pipelining
DataNodes from the NameNode. This list contains the DataNodes that will
host a replica of that block. The client then flushes the data block to
the first DataNode. The first DataNode starts receiving the data in
- small portions (4 KB), writes each portion to its local repository and
+ small portions, writes each portion to its local repository and
transfers that portion to the second DataNode in the list. The second
DataNode, in turn starts receiving each portion of the data block,
writes that portion to its repository and then flushes that portion to
@@ -416,7 +416,7 @@ Replication Pipelining
the next one in the pipeline. Thus, the data is pipelined from one
DataNode to the next.
-Accessibility
+* Accessibility
HDFS can be accessed from applications in many different ways.
Natively, HDFS provides a
@@ -426,7 +426,7 @@ Accessibility
of an HDFS instance. Work is in progress to expose HDFS through the WebDAV
protocol.
-FS Shell
+** FS Shell
HDFS allows user data to be organized in the form of files and
directories. It provides a commandline interface called FS shell that
@@ -447,7 +447,7 @@ FS Shell
FS shell is targeted for applications that need a scripting language to
interact with the stored data.
-DFSAdmin
+** DFSAdmin
The DFSAdmin command set is used for administering an HDFS cluster.
These are commands that are used only by an HDFS administrator. Here
@@ -463,16 +463,16 @@ DFSAdmin
|Recommission or decommission DataNode(s) | <<<bin/hadoop dfsadmin -refreshNodes>>>
*---------+---------+
-Browser Interface
+** Browser Interface
A typical HDFS install configures a web server to expose the HDFS
namespace through a configurable TCP port. This allows a user to
navigate the HDFS namespace and view the contents of its files using a
web browser.
-Space Reclamation
+* Space Reclamation
-File Deletes and Undeletes
+** File Deletes and Undeletes
When a file is deleted by a user or an application, it is not
immediately removed from HDFS. Instead, HDFS first renames it to a file
@@ -490,12 +490,12 @@ File Deletes and Undeletes
file. The <<</trash>>> directory contains only the latest copy of the file
that was deleted. The <<</trash>>> directory is just like any other directory
with one special feature: HDFS applies specified policies to
- automatically delete files from this directory. The current default
- policy is to delete files from <<</trash>>> that are more than 6 hours old.
- In the future, this policy will be configurable through a well defined
- interface.
+ automatically delete files from this directory. Current default trash
+ interval is set to 0 (Deletes file without storing in trash). This value is
+ configurable parameter stored as <<<fs.trash.interval>>> stored in
+ core-site.xml.
-Decrease Replication Factor
+** Decrease Replication Factor
When the replication factor of a file is reduced, the NameNode selects
excess replicas that can be deleted. The next Heartbeat transfers this
@@ -505,7 +505,7 @@ Decrease Replication Factor
of the setReplication API call and the appearance of free space in the
cluster.
-References
+* References
Hadoop {{{http://hadoop.apache.org/docs/current/api/}JavaDoc API}}.
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm Fri Jan 31 03:24:24 2014
@@ -869,7 +869,7 @@ Content-Length: 0
* {Error Responses}
When an operation fails, the server may throw an exception.
- The JSON schema of error responses is defined in {{{RemoteException JSON Schema}}}.
+ The JSON schema of error responses is defined in {{RemoteException JSON Schema}}.
The table below shows the mapping from exceptions to HTTP response codes.
** {HTTP Response Codes}
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java Fri Jan 31 03:24:24 2014
@@ -71,7 +71,7 @@ public class TestFSOutputSummer {
cleanupFile(name);
}
- /* create a file, write data with vairable amount of data */
+ /* create a file, write data with variable amount of data */
private void writeFile3(Path name) throws Exception {
FSDataOutputStream stm = fileSys.create(name, true,
fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, 4096),
@@ -103,6 +103,8 @@ public class TestFSOutputSummer {
stm.readFully(0, actual);
checkAndEraseData(actual, 0, expected, "Read Sanity Test");
stm.close();
+ // do a sanity check. Get the file checksum
+ fileSys.getFileChecksum(name);
}
private void cleanupFile(Path name) throws IOException {
@@ -112,13 +114,20 @@ public class TestFSOutputSummer {
}
/**
- * Test write opeation for output stream in DFS.
+ * Test write operation for output stream in DFS.
*/
@Test
public void testFSOutputSummer() throws Exception {
+ doTestFSOutputSummer("CRC32");
+ doTestFSOutputSummer("CRC32C");
+ doTestFSOutputSummer("NULL");
+ }
+
+ private void doTestFSOutputSummer(String checksumType) throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BYTES_PER_CHECKSUM);
+ conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, checksumType);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_OF_DATANODES)
.build();