You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2011/11/28 19:21:54 UTC
svn commit: r1207490 - in
/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common: ./
src/main/docs/ src/main/docs/changes/ src/main/java/
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/http/
src/main/java/org/apache/...
Author: atm
Date: Mon Nov 28 18:21:46 2011
New Revision: 1207490
URL: http://svn.apache.org/viewvc?rev=1207490&view=rev
Log:
Merge trunk into HA branch.
Added:
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
- copied unchanged from r1207485, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestStaticMapping.java
- copied unchanged from r1207485, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestStaticMapping.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java
- copied unchanged from r1207485, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestSwitchMapping.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov 28 18:21:46 2011
@@ -3,7 +3,12 @@ Hadoop Change Log
Trunk (unreleased changes)
INCOMPATIBLE CHANGES
-
+
+ NEW FEATURES
+
+ HADOOP-7777 Implement a base class for DNSToSwitchMapping implementations
+ that can offer extra topology information. (stevel)
+
IMPROVEMENTS
HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)
@@ -118,6 +123,9 @@ Release 0.23.1 - Unreleased
HADOOP-7802. Hadoop scripts unconditionally source
"$bin"/../libexec/hadoop-config.sh. (Bruno Mahé via tomwhite)
+ HADOOP-7858. Drop some info logging to DEBUG level in IPC,
+ metrics, and HTTP. (todd via eli)
+
OPTIMIZATIONS
BUG FIXES
@@ -131,6 +139,8 @@ Release 0.23.1 - Unreleased
HADOOP-6614. RunJar should provide more diags when it can't create
a temp file. (Jonathan Hsieh via eli)
+ HADOOP-7859. TestViewFsHdfs.testgetFileLinkStatus is failing an assert. (eli)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
@@ -1096,6 +1106,12 @@ Release 0.22.0 - Unreleased
HADOOP-7786. Remove HDFS-specific config keys defined in FsConfig. (eli)
+ HADOOP-7358. Improve log levels when exceptions caught in RPC handler
+ (Todd Lipcon via shv)
+
+ HADOOP-7861. changes2html.pl generates links to HADOOP, HDFS, and MAPREDUCE
+ jiras. (shv)
+
OPTIMIZATIONS
HADOOP-6884. Add LOG.isDebugEnabled() guard for each LOG.debug(..).
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,5 +1,5 @@
/hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1207485
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1207485
/hadoop/core/branches/branch-0.19/src/docs:713112
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl Mon Nov 28 18:21:46 2011
@@ -242,7 +242,11 @@ for my $rel (@releases) {
$item =~ s:\s*(\([^)"]+?\))\s*$:<br />$1:; # Separate attribution
$item =~ s:\n{2,}:\n<p/>\n:g; # Keep paragraph breaks
- $item =~ s{(?:${jira_url_prefix})?(HADOOP-\d+)} # Link to JIRA
+ $item =~ s{(?:${jira_url_prefix})?(HADOOP-\d+)} # Link to JIRA Common
+ {<a href="${jira_url_prefix}$1">$1</a>}g;
+ $item =~ s{(?:${jira_url_prefix})?(HDFS-\d+)} # Link to JIRA Hdfs
+ {<a href="${jira_url_prefix}$1">$1</a>}g;
+ $item =~ s{(?:${jira_url_prefix})?(MAPREDUCE-\d+)} # Link to JIRA MR
{<a href="${jira_url_prefix}$1">$1</a>}g;
print " <li>$item</li>\n";
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1207485
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Mon Nov 28 18:21:46 2011
@@ -1099,18 +1099,10 @@ public final class FileContext {
*/
private Path qualifySymlinkTarget(final AbstractFileSystem pathFS,
Path pathWithLink, Path target) {
- /* NB: makeQualified uses the target's scheme and authority, if
- * specified, and the scheme and authority of pathFS, if not. If
- * the path does have a scheme and authority we assert they match
- * those of pathFS since resolve updates the file system of a path
- * that contains links each time a link is encountered.
- */
+ // NB: makeQualified uses the target's scheme and authority, if
+ // specified, and the scheme and authority of pathFS, if not.
final String scheme = target.toUri().getScheme();
final String auth = target.toUri().getAuthority();
- if (scheme != null && auth != null) {
- assert scheme.equals(pathFS.getUri().getScheme());
- assert auth.equals(pathFS.getUri().getAuthority());
- }
return (scheme == null && auth == null)
? target.makeQualified(pathFS.getUri(), pathWithLink.getParent())
: target;
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java Mon Nov 28 18:21:46 2011
@@ -644,12 +644,12 @@ public class HttpServer implements Filte
while (true) {
try {
port = webServer.getConnectors()[0].getLocalPort();
- LOG.info("Port returned by webServer.getConnectors()[0]." +
+ LOG.debug("Port returned by webServer.getConnectors()[0]." +
"getLocalPort() before open() is "+ port +
". Opening the listener on " + oriPort);
listener.open();
port = listener.getLocalPort();
- LOG.info("listener.getLocalPort() returned " + listener.getLocalPort() +
+ LOG.debug("listener.getLocalPort() returned " + listener.getLocalPort() +
" webServer.getConnectors()[0].getLocalPort() returned " +
webServer.getConnectors()[0].getLocalPort());
//Workaround to handle the problem reported in HADOOP-4744
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Mon Nov 28 18:21:46 2011
@@ -1498,7 +1498,7 @@ public abstract class Server {
@Override
public void run() {
- LOG.info(getName() + ": starting");
+ LOG.debug(getName() + ": starting");
SERVER.set(Server.this);
ByteArrayOutputStream buf =
new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
@@ -1536,7 +1536,16 @@ public abstract class Server {
);
}
} catch (Throwable e) {
- LOG.info(getName() + ", call: " + call + ", error: ", e);
+ String logMsg = getName() + ", call " + call + ": error: " + e;
+ if (e instanceof RuntimeException || e instanceof Error) {
+ // These exception types indicate something is probably wrong
+ // on the server side, as opposed to just a normal exceptional
+ // result.
+ LOG.warn(logMsg, e);
+ } else {
+ LOG.info(logMsg, e);
+ }
+
errorClass = e.getClass().getName();
error = StringUtils.stringifyException(e);
// Remove redundant error class name from the beginning of the stack trace
@@ -1571,7 +1580,7 @@ public abstract class Server {
LOG.info(getName() + " caught an exception", e);
}
}
- LOG.info(getName() + ": exiting");
+ LOG.debug(getName() + ": exiting");
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java Mon Nov 28 18:21:46 2011
@@ -389,7 +389,7 @@ public class WritableRpcEngine implement
}
protocolImplMap.put(new ProtoNameVer(protocolName, version),
new ProtoClassProtoImpl(protocolClass, protocolImpl));
- LOG.info("Protocol Name = " + protocolName + " version=" + version +
+ LOG.debug("Protocol Name = " + protocolName + " version=" + version +
" ProtocolImpl=" + protocolImpl.getClass().getName() +
" protocolClass=" + protocolClass.getName());
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java Mon Nov 28 18:21:46 2011
@@ -241,7 +241,7 @@ public class MetricsSystemImpl extends M
injectedTags, period, config.subset(SOURCE_KEY));
sources.put(name, sa);
sa.start();
- LOG.info("Registered source "+ name);
+ LOG.debug("Registered source "+ name);
}
@Override public synchronized <T extends MetricsSink>
@@ -405,8 +405,8 @@ public class MetricsSystemImpl extends M
private synchronized void stopSources() {
for (Entry<String, MetricsSourceAdapter> entry : sources.entrySet()) {
MetricsSourceAdapter sa = entry.getValue();
- LOG.info("Stopping metrics source "+ entry.getKey());
- LOG.debug(sa.source().getClass());
+ LOG.debug("Stopping metrics source "+ entry.getKey() +
+ ": class=" + sa.source().getClass());
sa.stop();
}
sysSource.stop();
@@ -416,8 +416,8 @@ public class MetricsSystemImpl extends M
private synchronized void stopSinks() {
for (Entry<String, MetricsSinkAdapter> entry : sinks.entrySet()) {
MetricsSinkAdapter sa = entry.getValue();
- LOG.info("Stopping metrics sink "+ entry.getKey());
- LOG.debug(sa.sink().getClass());
+ LOG.debug("Stopping metrics sink "+ entry.getKey() +
+ ": class=" + sa.sink().getClass());
sa.stop();
}
sinks.clear();
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/CachedDNSToSwitchMapping.java Mon Nov 28 18:21:46 2011
@@ -34,9 +34,13 @@ import org.apache.hadoop.classification.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public class CachedDNSToSwitchMapping implements DNSToSwitchMapping {
+public class CachedDNSToSwitchMapping extends AbstractDNSToSwitchMapping {
private Map<String, String> cache = new ConcurrentHashMap<String, String>();
- protected DNSToSwitchMapping rawMapping;
+
+ /**
+ * The uncached mapping
+ */
+ protected final DNSToSwitchMapping rawMapping;
/**
* cache a raw DNS mapping
@@ -118,4 +122,14 @@ public class CachedDNSToSwitchMapping im
return getCachedHosts(names);
}
+
+ /**
+ * Delegate the switch topology query to the raw mapping, via
+ * {@link AbstractDNSToSwitchMapping#isMappingSingleSwitch(DNSToSwitchMapping)}
+ * @return true iff the raw mapper is considered single-switch.
+ */
+ @Override
+ public boolean isSingleSwitch() {
+ return isMappingSingleSwitch(rawMapping);
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java Mon Nov 28 18:21:46 2011
@@ -40,6 +40,12 @@ public interface DNSToSwitchMapping {
* Note the hostname/ip-address is not part of the returned path.
* The network topology of the cluster would determine the number of
* components in the network path.
+ * <p/>
+ *
+ * If a name cannot be resolved to a rack, the implementation
+ * should return {@link NetworkTopology#DEFAULT_RACK}. This
+ * is what the bundled implementations do, though it is not a formal requirement
+ *
* @param names the list of hosts to resolve (can be empty)
* @return list of resolved network paths.
* If <i>names</i> is empty, the returned list is also empty
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java Mon Nov 28 18:21:46 2011
@@ -32,16 +32,21 @@ import org.apache.hadoop.fs.CommonConfig
/**
* This class implements the {@link DNSToSwitchMapping} interface using a
- * script configured via the {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY}
+ * script configured via the
+ * {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
+ * <p/>
+ * It contains a static class <code>RawScriptBasedMapping</code> that performs
+ * the work: reading the configuration parameters, executing any defined
+ * script, handling errors and such like. The outer
+ * class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ * queries.
+ * <p/>
+ * This DNS mapper's {@link #isSingleSwitch()} predicate returns
+ * true if and only if a script is defined.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public final class ScriptBasedMapping extends CachedDNSToSwitchMapping
-implements Configurable
-{
- public ScriptBasedMapping() {
- super(new RawScriptBasedMapping());
- }
+public final class ScriptBasedMapping extends CachedDNSToSwitchMapping {
/**
* Minimum number of arguments: {@value}
@@ -66,6 +71,18 @@ implements Configurable
CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY ;
/**
+ * Create an instance with the default configuration.
+ * </p>
+ * Calling {@link #setConf(Configuration)} will trigger a
+ * re-evaluation of the configuration settings and so be used to
+ * set up the mapping script.
+ *
+ */
+ public ScriptBasedMapping() {
+ super(new RawScriptBasedMapping());
+ }
+
+ /**
* Create an instance from the given configuration
* @param conf configuration
*/
@@ -74,14 +91,31 @@ implements Configurable
setConf(conf);
}
+ /**
+ * Get the cached mapping and convert it to its real type
+ * @return the inner raw script mapping.
+ */
+ private RawScriptBasedMapping getRawMapping() {
+ return (RawScriptBasedMapping)rawMapping;
+ }
+
@Override
public Configuration getConf() {
- return ((RawScriptBasedMapping)rawMapping).getConf();
+ return getRawMapping().getConf();
}
+ /**
+ * {@inheritDoc}
+ * <p/>
+ * This will get called in the superclass constructor, so a check is needed
+ * to ensure that the raw mapping is defined before trying to relaying a null
+ * configuration.
+ * @param conf
+ */
@Override
public void setConf(Configuration conf) {
- ((RawScriptBasedMapping)rawMapping).setConf(conf);
+ super.setConf(conf);
+ getRawMapping().setConf(conf);
}
/**
@@ -89,29 +123,26 @@ implements Configurable
* by the superclass {@link CachedDNSToSwitchMapping}
*/
private static final class RawScriptBasedMapping
- implements DNSToSwitchMapping {
+ extends AbstractDNSToSwitchMapping {
private String scriptName;
- private Configuration conf;
private int maxArgs; //max hostnames per call of the script
- private static Log LOG =
+ private static final Log LOG =
LogFactory.getLog(ScriptBasedMapping.class);
/**
- * Set the configuration and
- * @param conf extract the configuration parameters of interest
+ * Set the configuration and extract the configuration parameters of interest
+ * @param conf the new configuration
*/
+ @Override
public void setConf (Configuration conf) {
- this.scriptName = conf.get(SCRIPT_FILENAME_KEY);
- this.maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
- this.conf = conf;
- }
-
- /**
- * Get the configuration
- * @return the configuration
- */
- public Configuration getConf () {
- return conf;
+ super.setConf(conf);
+ if (conf != null) {
+ scriptName = conf.get(SCRIPT_FILENAME_KEY);
+ maxArgs = conf.getInt(SCRIPT_ARG_COUNT_KEY, DEFAULT_ARG_COUNT);
+ } else {
+ scriptName = null;
+ maxArgs = 0;
+ }
}
/**
@@ -122,42 +153,42 @@ implements Configurable
@Override
public List<String> resolve(List<String> names) {
- List <String> m = new ArrayList<String>(names.size());
-
- if (names.isEmpty()) {
- return m;
- }
+ List<String> m = new ArrayList<String>(names.size());
- if (scriptName == null) {
- for (int i = 0; i < names.size(); i++) {
- m.add(NetworkTopology.DEFAULT_RACK);
+ if (names.isEmpty()) {
+ return m;
}
- return m;
- }
-
- String output = runResolveCommand(names);
- if (output != null) {
- StringTokenizer allSwitchInfo = new StringTokenizer(output);
- while (allSwitchInfo.hasMoreTokens()) {
- String switchInfo = allSwitchInfo.nextToken();
- m.add(switchInfo);
- }
-
- if (m.size() != names.size()) {
- // invalid number of entries returned by the script
- LOG.error("Script " + scriptName + " returned "
- + Integer.toString(m.size()) + " values when "
- + Integer.toString(names.size()) + " were expected.");
+
+ if (scriptName == null) {
+ for (String name : names) {
+ m.add(NetworkTopology.DEFAULT_RACK);
+ }
+ return m;
+ }
+
+ String output = runResolveCommand(names);
+ if (output != null) {
+ StringTokenizer allSwitchInfo = new StringTokenizer(output);
+ while (allSwitchInfo.hasMoreTokens()) {
+ String switchInfo = allSwitchInfo.nextToken();
+ m.add(switchInfo);
+ }
+
+ if (m.size() != names.size()) {
+ // invalid number of entries returned by the script
+ LOG.error("Script " + scriptName + " returned "
+ + Integer.toString(m.size()) + " values when "
+ + Integer.toString(names.size()) + " were expected.");
+ return null;
+ }
+ } else {
+ // an error occurred. return null to signify this.
+ // (exn was already logged in runResolveCommand)
return null;
}
- } else {
- // an error occurred. return null to signify this.
- // (exn was already logged in runResolveCommand)
- return null;
+
+ return m;
}
-
- return m;
- }
/**
* Build and execute the resolution command. The command is
@@ -195,10 +226,10 @@ implements Configurable
dir = new File(userDir);
}
ShellCommandExecutor s = new ShellCommandExecutor(
- cmdList.toArray(new String[0]), dir);
+ cmdList.toArray(new String[cmdList.size()]), dir);
try {
s.execute();
- allOutput.append(s.getOutput() + " ");
+ allOutput.append(s.getOutput()).append(" ");
} catch (Exception e) {
LOG.warn("Exception: ", e);
return null;
@@ -207,5 +238,15 @@ implements Configurable
}
return allOutput.toString();
}
+
+ /**
+ * Declare that the mapper is single-switched if a script was not named
+ * in the configuration.
+ * @return true iff there is no script
+ */
+ @Override
+ public boolean isSingleSwitch() {
+ return scriptName == null;
+ }
}
}
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Nov 28 18:21:46 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1204792
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1207485
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/StaticMapping.java Mon Nov 28 18:21:46 2011
@@ -17,34 +17,80 @@
*/
package org.apache.hadoop.net;
-import java.util.*;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* Implements the {@link DNSToSwitchMapping} via static mappings. Used
- * in testcases that simulate racks.
+ * in testcases that simulate racks, and in the
+ * {@link org.apache.hadoop.hdfs.MiniDFSCluster}
+ *
+ * A shared, static mapping is used; to reset it call {@link #resetMap()}.
*
+ * When an instance of the class has its {@link #setConf(Configuration)}
+ * method called, nodes listed in the configuration will be added to the map.
+ * These do not get removed when the instance is garbage collected.
*/
-public class StaticMapping extends Configured implements DNSToSwitchMapping {
- public void setconf(Configuration conf) {
- String[] mappings = conf.getStrings("hadoop.configured.node.mapping");
- if (mappings != null) {
- for (int i = 0; i < mappings.length; i++) {
- String str = mappings[i];
- String host = str.substring(0, str.indexOf('='));
- String rack = str.substring(str.indexOf('=') + 1);
- addNodeToRack(host, rack);
+public class StaticMapping extends AbstractDNSToSwitchMapping {
+
+ /**
+ * key to define the node mapping as a comma-delimited list of host=rack
+ * mappings, e.g. <code>host1=r1,host2=r1,host3=r2</code>.
+ * </p>
+ * <b>Important: </b>spaces not trimmed and are considered significant.
+ */
+ public static final String KEY_HADOOP_CONFIGURED_NODE_MAPPING =
+ "hadoop.configured.node.mapping";
+
+ /**
+ * Configure the mapping by extracting any mappings defined in the
+ * {@link #KEY_HADOOP_CONFIGURED_NODE_MAPPING} field
+ * @param conf new configuration
+ */
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ if (conf != null) {
+ String[] mappings = conf.getStrings(KEY_HADOOP_CONFIGURED_NODE_MAPPING);
+ if (mappings != null) {
+ for (String str : mappings) {
+ String host = str.substring(0, str.indexOf('='));
+ String rack = str.substring(str.indexOf('=') + 1);
+ addNodeToRack(host, rack);
+ }
}
}
}
+
+ /**
+ * retained lower case setter for compatibility reasons; relays to
+ * {@link #setConf(Configuration)}
+ * @param conf new configuration
+ */
+ public void setconf(Configuration conf) {
+ setConf(conf);
+ }
+
/* Only one instance per JVM */
- private static Map<String, String> nameToRackMap = new HashMap<String, String>();
-
- static synchronized public void addNodeToRack(String name, String rackId) {
- nameToRackMap.put(name, rackId);
+ private static final Map<String, String> nameToRackMap = new HashMap<String, String>();
+
+ /**
+ * Add a node to the static map. The moment any entry is added to the map,
+ * the map goes multi-rack.
+ * @param name node name
+ * @param rackId rack ID
+ */
+ public static void addNodeToRack(String name, String rackId) {
+ synchronized (nameToRackMap) {
+ nameToRackMap.put(name, rackId);
+ }
}
+
+ @Override
public List<String> resolve(List<String> names) {
List<String> m = new ArrayList<String>();
synchronized (nameToRackMap) {
@@ -59,4 +105,24 @@ public class StaticMapping extends Confi
return m;
}
}
+
+ /**
+ * This mapping is only single switch if the map is empty
+ * @return the current switching status
+ */
+ @Override
+ public boolean isSingleSwitch() {
+ synchronized (nameToRackMap) {
+ return nameToRackMap.isEmpty();
+ }
+ }
+
+ /**
+ * Clear the map and revert to being a single switch
+ */
+ public static void resetMap() {
+ synchronized (nameToRackMap) {
+ nameToRackMap.clear();
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java?rev=1207490&r1=1207489&r2=1207490&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestScriptBasedMapping.java Mon Nov 28 18:21:46 2011
@@ -23,30 +23,59 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import junit.framework.TestCase;
+import org.junit.Test;
public class TestScriptBasedMapping extends TestCase {
- private ScriptBasedMapping mapping;
- private Configuration conf;
- private List<String> names;
+
public TestScriptBasedMapping() {
- mapping = new ScriptBasedMapping();
-
- conf = new Configuration();
- conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
- ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
- conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
- mapping.setConf(conf);
}
+ @Test
public void testNoArgsMeansNoResult() {
- names = new ArrayList<String>();
+ Configuration conf = new Configuration();
+ conf.setInt(ScriptBasedMapping.SCRIPT_ARG_COUNT_KEY,
+ ScriptBasedMapping.MIN_ALLOWABLE_ARGS - 1);
+ conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
+ conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
+ ScriptBasedMapping mapping = createMapping(conf);
+ List<String> names = new ArrayList<String>();
names.add("some.machine.name");
names.add("other.machine.name");
List<String> result = mapping.resolve(names);
- assertNull(result);
+ assertNull("Expected an empty list", result);
}
+ @Test
+ public void testNoFilenameMeansSingleSwitch() throws Throwable {
+ Configuration conf = new Configuration();
+ ScriptBasedMapping mapping = createMapping(conf);
+ assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+ assertTrue("Expected to be single switch",
+ AbstractDNSToSwitchMapping.isMappingSingleSwitch(mapping));
+ }
+
+ @Test
+ public void testFilenameMeansMultiSwitch() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ScriptBasedMapping.SCRIPT_FILENAME_KEY, "any-filename");
+ ScriptBasedMapping mapping = createMapping(conf);
+ assertFalse("Expected to be multi switch", mapping.isSingleSwitch());
+ mapping.setConf(new Configuration());
+ assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+ }
+
+ @Test
+ public void testNullConfig() throws Throwable {
+ ScriptBasedMapping mapping = createMapping(null);
+ assertTrue("Expected to be single switch", mapping.isSingleSwitch());
+
+ }
+ private ScriptBasedMapping createMapping(Configuration conf) {
+ ScriptBasedMapping mapping = new ScriptBasedMapping();
+ mapping.setConf(conf);
+ return mapping;
+ }
}