You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2010/09/30 17:32:04 UTC
svn commit: r1003106 - in /hadoop/hdfs/branches/HDFS-1052: ./
src/c++/libhdfs/ src/contrib/hdfsproxy/
src/docs/src/documentation/content/xdocs/ src/java/
src/java/org/apache/hadoop/hdfs/
src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apa...
Author: suresh
Date: Thu Sep 30 15:32:03 2010
New Revision: 1003106
URL: http://svn.apache.org/viewvc?rev=1003106&view=rev
Log:
Merging changes from trunk
Added:
hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java
- copied unchanged from r1002971, hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/TestFiHftp.java
hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj
- copied unchanged from r1002971, hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/namenode/FileDataServletAspects.aj
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java
- copied unchanged from r1002971, hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java
Modified:
hadoop/hdfs/branches/HDFS-1052/ (props changed)
hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
hadoop/hdfs/branches/HDFS-1052/build.xml (props changed)
hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/docs/src/documentation/content/xdocs/tabs.xml
hadoop/hdfs/branches/HDFS-1052/src/java/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDatanodeDescriptor.java
hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/HDFS-1052/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-999162
+/hadoop/hdfs/trunk:987665-1002971
Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Thu Sep 30 15:32:03 2010
@@ -48,6 +48,8 @@ Trunk (unreleased changes)
IMPROVEMENTS
+ HDFS-1304. Add a new unit test for HftpFileSystem.open(..). (szetszwo)
+
HDFS-1096. fix for prev. commit. (boryas)
HDFS-1096. allow dfsadmin/mradmin refresh of superuser proxy group
@@ -147,6 +149,11 @@ Trunk (unreleased changes)
HDFS-1407. Change DataTransferProtocol methods to use Block instead
of individual elements of Block. (suresh)
+ HDFS-1417. Add @Override to SimulatedFSDataset methods that implement
+ FSDatasetInterface methods. (suresh)
+
+ HDFS-1426. Remove unused method BlockInfo#listCount. (hairong)
+
OPTIMIZATIONS
HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)
@@ -160,6 +167,8 @@ Trunk (unreleased changes)
HDFS-1320. Add LOG.isDebugEnabled() guard for each LOG.debug(..).
(Erik Steffl via szetszwo)
+ HDFS-1368. Add a block counter to DatanodeDescriptor. (hairong)
+
BUG FIXES
HDFS-1039. Adding test for JspHelper.getUGI(jnp via boryas)
@@ -281,7 +290,14 @@ Trunk (unreleased changes)
HDFS-1419. Federation: Three test cases need minor modification after
the new block id change (Tanping Wang via suresh)
-Release 0.21.0 - Unreleased
+ HDFS-96. HDFS supports blocks larger than 2 GB.
+ (Patrick Kling via dhruba)
+
+ HDFS-1364. Makes long running HFTP-based applications do relogins
+ if necessary. (Jitendra Pandey via ddas)
+
+ HDFS-1399. Distinct minicluster services (e.g. NN and JT) overwrite each
+ other's service policies. (Aaron T. Myers via tomwhite)
INCOMPATIBLE CHANGES
@@ -1210,6 +1226,8 @@ Release 0.21.0 - Unreleased
HDFS-1363. Eliminate second synchronized sections in appendFile(). (shv)
+ HDFS-1413. Fix broken links to HDFS Wiki. (shv)
+
Release 0.20.3 - Unreleased
IMPROVEMENTS
Propchange: hadoop/hdfs/branches/HDFS-1052/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
/hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:987665-999162
+/hadoop/hdfs/trunk/build.xml:987665-1002971
Propchange: hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-999162
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1002971
Propchange: hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-999162
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1002971
Modified: hadoop/hdfs/branches/HDFS-1052/src/docs/src/documentation/content/xdocs/tabs.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/docs/src/documentation/content/xdocs/tabs.xml?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/docs/src/documentation/content/xdocs/tabs.xml (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/docs/src/documentation/content/xdocs/tabs.xml Thu Sep 30 15:32:03 2010
@@ -31,7 +31,7 @@
-->
<tab label="Project" href="http://hadoop.apache.org/hdfs/" />
- <tab label="Wiki" href="http://wiki.apache.org/hadoop/hdfs" />
+ <tab label="Wiki" href="http://wiki.apache.org/hadoop/HDFS" />
<tab label="HDFS 0.22 Documentation" dir="" />
</tabs>
Propchange: hadoop/hdfs/branches/HDFS-1052/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-999162
+/hadoop/hdfs/trunk/src/java:987665-1002971
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSInputStream.java Thu Sep 30 15:32:03 2010
@@ -524,7 +524,7 @@ public class DFSInputStream extends FSIn
if (pos > blockEnd) {
currentNode = blockSeekTo(pos);
}
- int realLen = Math.min(len, (int) (blockEnd - pos + 1));
+ int realLen = (int) Math.min((long) len, (blockEnd - pos + 1L));
int result = readBuffer(buf, off, realLen);
if (result >= 0) {
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/HftpFileSystem.java Thu Sep 30 15:32:03 2010
@@ -178,6 +178,10 @@ public class HftpFileSystem extends File
break;
}
}
+
+ //Renew TGT if needed
+ ugi.reloginFromKeytab();
+
//since we don't already have a token, go get one over https
if (delegationToken == null) {
delegationToken =
@@ -659,6 +663,7 @@ public class HftpFileSystem extends File
final HftpFileSystem fs = weakFs.get();
if (fs != null) {
synchronized (fs) {
+ fs.ugi.reloginFromKeytab();
fs.ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java Thu Sep 30 15:32:03 2010
@@ -300,8 +300,8 @@ class BlockSender implements java.io.Clo
throws IOException {
// Sends multiple chunks in one packet with a single write().
- int len = Math.min((int) (endOffset - offset),
- bytesPerChecksum*maxChunks);
+ int len = (int) Math.min(endOffset - offset,
+ (((long) bytesPerChecksum) * ((long) maxChunks)));
int numChunks = (len + bytesPerChecksum - 1)/bytesPerChecksum;
int packetLen = len + numChunks*checksumSize + 4;
boolean lastDataPacket = offset + len == endOffset && len > 0;
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Thu Sep 30 15:32:03 2010
@@ -54,6 +54,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
@@ -112,7 +113,6 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Daemon;
@@ -457,12 +457,6 @@ public class DataNode extends Configured
// adjust info port
this.dnRegistration.setInfoPort(this.infoServer.getPort());
myMetrics = new DataNodeMetrics(conf, dnRegistration.getName());
-
- // set service-level authorization security policy
- if (conf.getBoolean(
- ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false)) {
- ServiceAuthorizationManager.refresh(conf, new HDFSPolicyProvider());
- }
// BlockTokenSecretManager is created here, but it shouldn't be
// used until it is initialized in register().
@@ -474,7 +468,13 @@ public class DataNode extends Configured
ipcServer = RPC.getServer(DataNode.class, this, ipcAddr.getHostName(),
ipcAddr.getPort(), conf.getInt("dfs.datanode.handler.count", 3), false,
conf, blockTokenSecretManager);
-
+
+ // set service-level authorization security policy
+ if (conf.getBoolean(
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
+ ipcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ }
+
dnRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());
LOG.info("dnRegistration = " + dnRegistration);
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BlockInfo.java Thu Sep 30 15:32:03 2010
@@ -249,14 +249,6 @@ class BlockInfo extends Block implements
return head;
}
- int listCount(DatanodeDescriptor dn) {
- int count = 0;
- for(BlockInfo cur = this; cur != null;
- cur = cur.getNext(cur.findDatanode(dn)))
- count++;
- return count;
- }
-
boolean listIsConsistent(DatanodeDescriptor dn) {
// going forward
int count = 0;
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java Thu Sep 30 15:32:03 2010
@@ -101,6 +101,7 @@ public class DatanodeDescriptor extends
}
private volatile BlockInfo blockList = null;
+ private int numBlocks = 0;
// isAlive == heartbeats.contains(this)
// This is an optimization, because contains takes O(n) time on Arraylist
protected boolean isAlive = false;
@@ -202,6 +203,7 @@ public class DatanodeDescriptor extends
return false;
// add to the head of the data-node list
blockList = b.listInsert(blockList, this);
+ numBlocks++;
return true;
}
@@ -211,7 +213,12 @@ public class DatanodeDescriptor extends
*/
boolean removeBlock(BlockInfo b) {
blockList = b.listRemove(blockList, this);
- return b.removeNode(this);
+ if ( b.removeNode(this) ) {
+ numBlocks--;
+ return true;
+ } else {
+ return false;
+ }
}
/**
@@ -247,7 +254,7 @@ public class DatanodeDescriptor extends
}
public int numBlocks() {
- return blockList == null ? 0 : blockList.listCount(this);
+ return numBlocks;
}
/**
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Thu Sep 30 15:32:03 2010
@@ -33,6 +33,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Options;
@@ -93,7 +94,6 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
-import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ServicePlugin;
@@ -165,8 +165,8 @@ public class NameNode implements Namenod
protected FSNamesystem namesystem;
protected NamenodeRole role;
- /** RPC server. */
- protected Server server;
+ /** RPC server. Package-protected for use in tests. */
+ Server server;
/** RPC server for HDFS Services communication.
BackupNode, Datanodes and all other services
should be connecting to this server if it is
@@ -347,13 +347,6 @@ public class NameNode implements Namenod
SecurityUtil.login(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, socAddr.getHostName());
int handlerCount = conf.getInt("dfs.namenode.handler.count", 10);
-
- // set service-level authorization security policy
- if (serviceAuthEnabled =
- conf.getBoolean(
- ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG, false)) {
- ServiceAuthorizationManager.refresh(conf, new HDFSPolicyProvider());
- }
NameNode.initMetrics(conf, this.getRole());
loadNamesystem(conf);
@@ -373,6 +366,17 @@ public class NameNode implements Namenod
socAddr.getHostName(), socAddr.getPort(),
handlerCount, false, conf,
namesystem.getDelegationTokenSecretManager());
+
+ // set service-level authorization security policy
+ if (serviceAuthEnabled =
+ conf.getBoolean(
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
+ this.server.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ if (this.serviceRpcServer != null) {
+ this.serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
+ }
+ }
+
// The rpc-server port can be ephemeral... ensure we have the correct info
this.rpcAddress = this.server.getListenerAddress();
setRpcServerAddress(conf);
@@ -1434,8 +1438,10 @@ public class NameNode implements Namenod
throw new AuthorizationException("Service Level Authorization not enabled!");
}
- ServiceAuthorizationManager.refresh(
- new Configuration(), new HDFSPolicyProvider());
+ this.server.refreshServiceAcl(new Configuration(), new HDFSPolicyProvider());
+ if (this.serviceRpcServer != null) {
+ this.serviceRpcServer.refreshServiceAcl(new Configuration(), new HDFSPolicyProvider());
+ }
}
@Override
Propchange: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-999162
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1002971
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Thu Sep 30 15:32:03 2010
@@ -66,7 +66,7 @@ public class SimulatedFSDataset impleme
"dfs.datanode.simulateddatastorage";
public static final String CONFIG_PROPERTY_CAPACITY =
"dfs.datanode.simulateddatastorage.capacity";
-
+//
public static final long DEFAULT_CAPACITY = 2L<<40; // 1 terabyte
public static final byte DEFAULT_DATABYTE = 9; // 1 terabyte
byte simulatedDataByte = DEFAULT_DATABYTE;
@@ -303,9 +303,6 @@ public class SimulatedFSDataset impleme
setConf(conf);
}
- private SimulatedFSDataset() { // real construction when setConf called.. Uggg
- }
-
public Configuration getConf() {
return conf;
}
@@ -347,6 +344,7 @@ public class SimulatedFSDataset impleme
}
}
+ @Override
public synchronized void finalizeBlock(Block b) throws IOException {
BInfo binfo = blockMap.get(b);
if (binfo == null) {
@@ -356,12 +354,14 @@ public class SimulatedFSDataset impleme
}
+ @Override
public synchronized void unfinalizeBlock(Block b) throws IOException {
if (isBeingWritten(b)) {
blockMap.remove(b);
}
}
+ @Override
public synchronized BlockListAsLongs getBlockReport() {
Block[] blockTable = new Block[blockMap.size()];
int count = 0;
@@ -389,6 +389,7 @@ public class SimulatedFSDataset impleme
return storage.getFree();
}
+ @Override
public synchronized long getLength(Block b) throws IOException {
BInfo binfo = blockMap.get(b);
if (binfo == null) {
@@ -403,7 +404,7 @@ public class SimulatedFSDataset impleme
return blockMap.get(new Block(blockId));
}
- /** {@inheritDoc} */
+ @Override
public Block getStoredBlock(long blkid) throws IOException {
Block b = new Block(blkid);
BInfo binfo = blockMap.get(b);
@@ -415,6 +416,7 @@ public class SimulatedFSDataset impleme
return b;
}
+ @Override
public synchronized void invalidate(Block[] invalidBlks) throws IOException {
boolean error = false;
if (invalidBlks == null) {
@@ -438,6 +440,7 @@ public class SimulatedFSDataset impleme
}
}
+ @Override
public synchronized boolean isValidBlock(Block b) {
// return (blockMap.containsKey(b));
BInfo binfo = blockMap.get(b);
@@ -545,6 +548,7 @@ public class SimulatedFSDataset impleme
return binfo;
}
+ @Override
public synchronized InputStream getBlockInputStream(Block b)
throws IOException {
BInfo binfo = blockMap.get(b);
@@ -556,6 +560,7 @@ public class SimulatedFSDataset impleme
return binfo.getIStream();
}
+ @Override
public synchronized InputStream getBlockInputStream(Block b, long seekOffset)
throws IOException {
InputStream result = getBlockInputStream(b);
@@ -564,6 +569,7 @@ public class SimulatedFSDataset impleme
}
/** Not supported */
+ @Override
public BlockInputStreams getTmpInputStreams(Block b, long blkoff, long ckoff
) throws IOException {
throw new IOException("Not supported");
@@ -588,7 +594,8 @@ public class SimulatedFSDataset impleme
}
return binfo.getMetaIStream();
}
-
+
+ @Override
public synchronized long getMetaDataLength(Block b) throws IOException {
BInfo binfo = blockMap.get(b);
if (binfo == null) {
@@ -601,6 +608,7 @@ public class SimulatedFSDataset impleme
return binfo.getMetaIStream().getLength();
}
+ @Override
public MetaDataInputStream getMetaDataInputStream(Block b)
throws IOException {
@@ -608,6 +616,7 @@ public class SimulatedFSDataset impleme
getMetaDataLength(b));
}
+ @Override
public synchronized boolean metaFileExists(Block b) throws IOException {
if (!isValidBlock(b)) {
throw new IOException("Block " + b +
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Thu Sep 30 15:32:03 2010
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na
import java.io.IOException;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
/**
@@ -48,4 +49,12 @@ public class NameNodeAdapter {
public static void refreshBlockCounts(NameNode namenode) {
namenode.getNamesystem().blockManager.updateState();
}
-}
\ No newline at end of file
+
+ /**
+ * Get the internal RPC server instance.
+ * @return rpc server
+ */
+ public static Server getRpcServer(NameNode namenode) {
+ return namenode.server;
+ }
+}
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDatanodeDescriptor.java?rev=1003106&r1=1003105&r2=1003106&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDatanodeDescriptor.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestDatanodeDescriptor.java Thu Sep 30 15:32:03 2010
@@ -48,4 +48,29 @@ public class TestDatanodeDescriptor exte
bc = dd.getInvalidateBlocks(MAX_LIMIT);
assertEquals(bc.getBlocks().length, REMAINING_BLOCKS);
}
+
+ public void testBlocksCounter() throws Exception {
+ DatanodeDescriptor dd = new DatanodeDescriptor();
+ assertEquals(0, dd.numBlocks());
+ BlockInfo blk = new BlockInfo(new Block(1L), 1);
+ BlockInfo blk1 = new BlockInfo(new Block(2L), 2);
+ // add first block
+ assertTrue(dd.addBlock(blk));
+ assertEquals(1, dd.numBlocks());
+ // remove a non-existent block
+ assertFalse(dd.removeBlock(blk1));
+ assertEquals(1, dd.numBlocks());
+ // add an existent block
+ assertFalse(dd.addBlock(blk));
+ assertEquals(1, dd.numBlocks());
+ // add second block
+ assertTrue(dd.addBlock(blk1));
+ assertEquals(2, dd.numBlocks());
+ // remove first block
+ assertTrue(dd.removeBlock(blk));
+ assertEquals(1, dd.numBlocks());
+ // remove second block
+ assertTrue(dd.removeBlock(blk1));
+ assertEquals(0, dd.numBlocks());
+ }
}
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-999162
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1002971
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-999162
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1002971
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Sep 30 15:32:03 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-999162
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1002971