You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2009/08/26 23:41:22 UTC
svn commit: r808193 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/
src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/test/...
Author: szetszwo
Date: Wed Aug 26 21:41:21 2009
New Revision: 808193
URL: http://svn.apache.org/viewvc?rev=808193&view=rev
Log:
HDFS-563. Simplify the codes in FSNamesystem.getBlockLocations(..).
Added:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeReport.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Wed Aug 26 21:41:21 2009
@@ -114,6 +114,9 @@
to be executed by the run-test-hdfs-fault-inject target. (Konstantin
Boudnik via szetszwo)
+ HDFS-563. Simplify the codes in FSNamesystem.getBlockLocations(..).
+ (szetszwo)
+
BUG FIXES
HDFS-76. Better error message to users when commands fail because of
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Aug 26 21:41:21 2009
@@ -641,8 +641,7 @@
/**
* Get block locations within the specified range.
- *
- * @see #getBlockLocations(String, long, long)
+ * @see ClientProtocol#getBlockLocations(String, long, long)
*/
LocatedBlocks getBlockLocations(String clientMachine, String src,
long offset, long length) throws IOException {
@@ -665,18 +664,9 @@
/**
* Get block locations within the specified range.
* @see ClientProtocol#getBlockLocations(String, long, long)
- */
- public LocatedBlocks getBlockLocations(String src, long offset, long length
- ) throws IOException {
- return getBlockLocations(src, offset, length, false);
- }
-
- /**
- * Get block locations within the specified range.
- * @see ClientProtocol#getBlockLocations(String, long, long)
* @throws FileNotFoundException
*/
- public LocatedBlocks getBlockLocations(String src, long offset, long length,
+ LocatedBlocks getBlockLocations(String src, long offset, long length,
boolean doAccessTime) throws IOException {
if (offset < 0) {
throw new IOException("Negative offset is not supported. File: " + src );
@@ -688,7 +678,7 @@
if (inode == null)
throw new FileNotFoundException();
final LocatedBlocks ret = getBlockLocationsInternal(src, inode,
- offset, length, Integer.MAX_VALUE, doAccessTime);
+ offset, length, doAccessTime);
if (auditLog.isInfoEnabled()) {
logAuditEvent(UserGroupInformation.getCurrentUGI(),
Server.getRemoteIp(),
@@ -701,25 +691,18 @@
INodeFile inode,
long offset,
long length,
- int nrBlocksToReturn,
boolean doAccessTime
) throws IOException {
- if(inode == null) {
- return null;
- }
if (doAccessTime && isAccessTimeSupported()) {
dir.setTimes(src, inode, -1, now(), false);
}
- Block[] blocks = inode.getBlocks();
+ final Block[] blocks = inode.getBlocks();
if (blocks == null) {
return null;
}
- if (blocks.length == 0) {
- return inode.createLocatedBlocks(new ArrayList<LocatedBlock>(blocks.length));
- }
-
- List<LocatedBlock> results = blockManager.getBlockLocations(blocks,
- offset, length, nrBlocksToReturn);
+ final List<LocatedBlock> results = blocks.length == 0?
+ new ArrayList<LocatedBlock>(0):
+ blockManager.getBlockLocations(blocks, offset, length, Integer.MAX_VALUE);
return inode.createLocatedBlocks(results);
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Wed Aug 26 21:41:21 2009
@@ -174,7 +174,7 @@
/** Return the {@link FSNamesystem} object.
* @return {@link FSNamesystem} object.
*/
- public FSNamesystem getNamesystem() {
+ FSNamesystem getNamesystem() {
return namesystem;
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Wed Aug 26 21:41:21 2009
@@ -19,18 +19,19 @@
import java.io.File;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
import java.net.URI;
+import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Collection;
-import java.nio.channels.FileChannel;
import java.util.Random;
-import java.io.RandomAccessFile;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
@@ -38,12 +39,15 @@
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.FSDatasetInterface;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.security.*;
+import org.apache.hadoop.net.DNSToSwitchMapping;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.StaticMapping;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -515,7 +519,7 @@
* @return {@link FSNamesystem} object.
*/
public FSNamesystem getNamesystem() {
- return nameNode.getNamesystem();
+ return NameNodeAdapter.getNamesystem(nameNode);
}
/**
@@ -916,7 +920,7 @@
* Set the softLimit and hardLimit of client lease periods
*/
void setLeasePeriod(long soft, long hard) {
- final FSNamesystem namesystem = nameNode.getNamesystem();
+ final FSNamesystem namesystem = getNamesystem();
namesystem.leaseManager.setLeasePeriod(soft, hard);
namesystem.lmthread.interrupt();
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeReport.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeReport.java?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeReport.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDatanodeReport.java Wed Aug 26 21:41:21 2009
@@ -78,7 +78,7 @@
Thread.sleep(5000);
FSNamesystemMetrics fsMetrics =
- cluster.getNameNode().getNamesystem().getFSNamesystemMetrics();
+ cluster.getNamesystem().getFSNamesystemMetrics();
assertEquals(1,fsMetrics.numExpiredHeartbeats.getCurrentIntervalValue());
}finally {
cluster.shutdown();
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDecommission.java Wed Aug 26 21:41:21 2009
@@ -34,7 +34,7 @@
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
/**
* This class tests the decommissioning of nodes.
@@ -158,7 +158,7 @@
/*
* decommission one random node.
*/
- private String decommissionNode(NameNode namenode,
+ private String decommissionNode(FSNamesystem namesystem,
Configuration conf,
DFSClient client,
FileSystem localFileSys)
@@ -183,7 +183,7 @@
ArrayList<String> nodes = new ArrayList<String>(decommissionedNodes);
nodes.add(nodename);
writeConfigFile(localFileSys, excludeFile, nodes);
- namenode.getNamesystem().refreshNodes(conf);
+ namesystem.refreshNodes(conf);
return nodename;
}
@@ -277,7 +277,7 @@
replicas + " replicas.");
checkFile(fileSys, file1, replicas);
printFileLocations(fileSys, file1);
- String downnode = decommissionNode(cluster.getNameNode(), conf,
+ String downnode = decommissionNode(cluster.getNamesystem(), conf,
client, localFileSys);
decommissionedNodes.add(downnode);
waitNodeState(fileSys, downnode, NodeState.DECOMMISSIONED);
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Wed Aug 26 21:41:21 2009
@@ -34,6 +34,7 @@
import org.apache.hadoop.hdfs.protocol.DataTransferProtocol;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessToken;
@@ -95,8 +96,8 @@
DFSTestUtil.waitReplication(fs, fileName, (short)1);
// get the block belonged to the created file
- LocatedBlocks blocks = cluster.getNamesystem().getBlockLocations(
- fileName.toString(), 0, (long)fileLen);
+ LocatedBlocks blocks = NameNodeAdapter.getBlockLocations(
+ cluster.getNameNode(), fileName.toString(), 0, (long)fileLen);
assertEquals(blocks.locatedBlockCount(), 1);
LocatedBlock block = blocks.get(0);
Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=808193&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (added)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Wed Aug 26 21:41:21 2009
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+
+/**
+ * This is a utility class to expose NameNode functionality for unit tests.
+ */
+public class NameNodeAdapter {
+ /**
+ * Get the namesystem from the namenode
+ */
+ public static FSNamesystem getNamesystem(NameNode namenode) {
+ return namenode.getNamesystem();
+ }
+
+ /**
+ * Get block locations within the specified range.
+ */
+ public static LocatedBlocks getBlockLocations(NameNode namenode,
+ String src, long offset, long length) throws IOException {
+ return namenode.getNamesystem().getBlockLocations(
+ src, offset, length, false);
+ }
+}
\ No newline at end of file
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java?rev=808193&r1=808192&r2=808193&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java Wed Aug 26 21:41:21 2009
@@ -30,6 +30,7 @@
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.namenode.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
/**
* Test for metrics published by the Namenode
@@ -53,7 +54,7 @@
protected void setUp() throws Exception {
cluster = new MiniDFSCluster(CONF, 3, true, null);
cluster.waitActive();
- namesystem = cluster.getNameNode().getNamesystem();
+ namesystem = cluster.getNamesystem();
fs = (DistributedFileSystem) cluster.getFileSystem();
metrics = namesystem.getFSNamesystemMetrics();
}
@@ -106,7 +107,8 @@
createFile(file, 100, (short)2);
// Corrupt first replica of the block
- LocatedBlock block = namesystem.getBlockLocations(file, 0, 1).get(0);
+ LocatedBlock block = NameNodeAdapter.getBlockLocations(
+ cluster.getNameNode(), file, 0, 1).get(0);
namesystem.markBlockAsCorrupt(block.getBlock(), block.getLocations()[0]);
updateMetrics();
assertEquals(1, metrics.corruptBlocks.get());
@@ -140,7 +142,8 @@
createFile(file, 100, (short)1);
// Corrupt the only replica of the block to result in a missing block
- LocatedBlock block = namesystem.getBlockLocations(file, 0, 1).get(0);
+ LocatedBlock block = NameNodeAdapter.getBlockLocations(
+ cluster.getNameNode(), file, 0, 1).get(0);
namesystem.markBlockAsCorrupt(block.getBlock(), block.getLocations()[0]);
updateMetrics();
assertEquals(1, metrics.underReplicatedBlocks.get());