You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by bo...@apache.org on 2011/03/15 00:05:56 UTC
svn commit: r1081603 - in /hadoop/hdfs/branches/HDFS-1052: ./
src/java/org/apache/hadoop/hdfs/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/java/org/apache/hadoop/hdfs/tools/
Author: boryas
Date: Mon Mar 14 23:05:56 2011
New Revision: 1081603
URL: http://svn.apache.org/viewvc?rev=1081603&view=rev
Log:
HDFS-1754. HDFS-1754. Federation: testFsck fails. (boryas)
Modified:
hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java
Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1081603&r1=1081602&r2=1081603&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Mon Mar 14 23:05:56 2011
@@ -241,6 +241,8 @@ Trunk (unreleased changes)
HDFS-1749. Federation: TestListCorruptFileBlocks failing in federation
branch. (jitendra)
+ HDFS-1754. Federation: testFsck fails. (boryas)
+
IMPROVEMENTS
HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1081603&r1=1081602&r2=1081603&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/DFSUtil.java Mon Mar 14 23:05:56 2011
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.security.UserGroupInformation;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
@InterfaceAudience.Private
@@ -441,6 +442,44 @@ public class DFSUtil {
// client should try {@link isDefaultNamenodeAddress} instead
return null;
}
+
+ /**
+ * return HTTP server info from the configuration
+ * @param conf
+ * @param namenode - namenode address
+ * @return http server info
+ */
+ public static String getInfoServer(
+ InetSocketAddress namenode, Configuration conf) {
+ String httpAddress = null;
+
+ String httpAddressKey = UserGroupInformation.isSecurityEnabled() ?
+ DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY
+ : DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+ String httpAddressDefault = UserGroupInformation.isSecurityEnabled() ?
+ DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT
+ :DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
+
+ if(namenode != null) {
+ // if non-default namenode, try reverse look up
+ // the nameServiceID if it is available
+ String nameServiceId = DFSUtil.getNameServiceIdFromAddress(
+ conf, namenode,
+ DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
+ DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+
+ if (nameServiceId != null) {
+ httpAddress = conf.get(DFSUtil.getNameServiceIdKey(
+ httpAddressKey, nameServiceId));
+ }
+ }
+ // else - Use non-federation style configuration
+ if (httpAddress == null) {
+ httpAddress = conf.get(httpAddressKey, httpAddressDefault);
+ }
+
+ return httpAddress;
+ }
/**
* Given the InetSocketAddress for any configured communication with a
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1081603&r1=1081602&r2=1081603&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Mon Mar 14 23:05:56 2011
@@ -447,14 +447,6 @@ public class NameNode implements Namenod
this.emptier.setDaemon(true);
this.emptier.start();
}
-
- public static String getInfoServer(Configuration conf) {
- return UserGroupInformation.isSecurityEnabled() ? conf.get(
- DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY,
- DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT) : conf.get(
- DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
- DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT);
- }
private void startHttpServer(final Configuration conf) throws IOException {
final InetSocketAddress infoSocAddr = getHttpServerAddress(conf);
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1081603&r1=1081602&r2=1081603&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Mon Mar 14 23:05:56 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtil.ErrorSimulator;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -408,7 +409,7 @@ public class SecondaryNameNode implement
throw new IOException("This is not a DFS");
}
- String configuredAddress = NameNode.getInfoServer(conf);
+ String configuredAddress = DFSUtil.getInfoServer(null, conf);
InetSocketAddress sockAddr = NetUtils.createSocketAddr(configuredAddress);
if (sockAddr.getAddress().isAnyLocalAddress()) {
if(UserGroupInformation.isSecurityEnabled()) {
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java?rev=1081603&r1=1081602&r2=1081603&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java Mon Mar 14 23:05:56 2011
@@ -199,16 +199,16 @@ public class DFSck extends Configured im
return errCode;
}
- /*
+ /**
* Derive the namenode http address from the current file system,
* either default or as set by "-fs" in the generic options.
- * Returns null if failure.
+ * @return Returns http address or null if failure.
*/
private String getCurrentNamenodeAddress() {
- String nnAddress = null;
+ //String nnAddress = null;
Configuration conf = getConf();
- //get the filesystem object
+ //get the filesystem object to verify it is an HDFS system
FileSystem fs;
try {
fs = FileSystem.get(conf);
@@ -227,41 +227,10 @@ public class DFSck extends Configured im
// The URI may have been provided by a human, and the server name may be
// aliased, so compare InetSocketAddresses instead of URI strings, and
// test against both possible variants of RPC address.
- InetSocketAddress namenode = NameNode.getAddress(dfs.getUri().getAuthority());
- String nameServiceId = DFSUtil.getNameServiceIdFromAddress(
- conf, namenode,
- DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
- DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
+ InetSocketAddress namenode =
+ NameNode.getAddress(dfs.getUri().getAuthority());
- //Look up nameservice-specific http address
- String httpAddressKey = UserGroupInformation.isSecurityEnabled() ?
- DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY
- : DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
- if (nameServiceId != null) {
- nnAddress = conf.get(DFSUtil.getNameServiceIdKey(
- httpAddressKey, nameServiceId));
- if (nnAddress != null)
- return nnAddress;
- else {
- System.err.println("Nameservice value for "
- + DFSUtil.getNameServiceIdKey(httpAddressKey, nameServiceId)
- + " expected but not available. Trying generic value.");
- //and fall through to next block
- }
- }
- // If that didn't work, check to see whether the filesystem URI addresses
- // the default namenode. If so, look up generic http address.
- boolean isDefaultNamenode = DFSUtil.isDefaultNamenodeAddress(
- conf, namenode,
- DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
- DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
- if (isDefaultNamenode) {
- return NameNode.getInfoServer(conf);
- } else {
- System.err.println("Cannot derive an unambiguous value for "
- + httpAddressKey + " from FileSystem " + fs.getUri());
- return null;
- }
+ return DFSUtil.getInfoServer(namenode, conf);
}
private int doWork(final String[] args) throws IOException {