You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2012/10/27 06:30:41 UTC
svn commit: r1402741 - in /hadoop/common/branches/branch-1: ./ src/core/
src/core/org/apache/hadoop/fs/
src/hdfs/org/apache/hadoop/hdfs/server/datanode/
src/mapred/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/hdfs/server/datanode/ src/test/org/...
Author: eli
Date: Sat Oct 27 04:30:41 2012
New Revision: 1402741
URL: http://svn.apache.org/viewvc?rev=1402741&view=rev
Log:
HADOOP-8968. Add a flag to completely disable the worker version check. Contributed by Alejandro Abdelnur
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/core/core-default.xml
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1402741&r1=1402740&r2=1402741&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Sat Oct 27 04:30:41 2012
@@ -107,6 +107,9 @@ Release 1.2.0 - unreleased
HADOOP-8567. Port conf servlet to dump running configuration to branch 1.x.
(Jing Zhao via suresh)
+ HADOOP-8968. Add a flag to completely disable the worker version check.
+ (tucu via eli)
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Modified: hadoop/common/branches/branch-1/src/core/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/core-default.xml?rev=1402741&r1=1402740&r2=1402741&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/core-default.xml (original)
+++ hadoop/common/branches/branch-1/src/core/core-default.xml Sat Oct 27 04:30:41 2012
@@ -578,6 +578,20 @@
</property>
<property>
+ <name>hadoop.skip.worker.version.check</name>
+ <value>false</value>
+ <description>
+ By default datanodes refuse to connect to namenodes if their build
+ revision (svn revision) do not match, and tasktrackers refuse to
+ connect to jobtrackers if their build version (version, revision,
+ user, and source checksum) do not match. This option changes the
+ behavior of hadoop workers to skip doing a version check at all.
+ This option supersedes the 'hadoop.relaxed.worker.version.check'
+ option.
+ </description>
+</property>
+
+<property>
<name>hadoop.jetty.logs.serve.aliases</name>
<value>true</value>
<description>
Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1402741&r1=1402740&r2=1402741&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java Sat Oct 27 04:30:41 2012
@@ -74,6 +74,11 @@ public class CommonConfigurationKeys {
"hadoop.relaxed.worker.version.check";
public static final boolean HADOOP_RELAXED_VERSION_CHECK_DEFAULT = false;
+ /** See src/core/core-default.xml */
+ public static final String HADOOP_SKIP_VERSION_CHECK_KEY =
+ "hadoop.skip.worker.version.check";
+ public static final boolean HADOOP_SKIP_VERSION_CHECK_DEFAULT = false;
+
/** Enable/Disable aliases serving from jetty */
public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES =
"hadoop.jetty.logs.serve.aliases";
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1402741&r1=1402740&r2=1402741&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java Sat Oct 27 04:30:41 2012
@@ -77,11 +77,9 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
-import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
-import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.datanode.FSDataset.VolumeInfo;
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeInstrumentation;
@@ -244,6 +242,11 @@ public class DataNode extends Configured
private boolean relaxedVersionCheck;
/**
+ * Whether the DN completely skips version check with the NN.
+ */
+ private boolean noVersionCheck;
+
+ /**
* Testing hook that allows tests to delay the sending of blockReceived RPCs
* to the namenode. This can help find bugs in append.
*/
@@ -364,6 +367,9 @@ public class DataNode extends Configured
this.relaxedVersionCheck = conf.getBoolean(
CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY,
CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_DEFAULT);
+ noVersionCheck = conf.getBoolean(
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY,
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_DEFAULT);
InetSocketAddress socAddr = DataNode.getStreamingAddr(conf);
int tmpPort = socAddr.getPort();
@@ -600,22 +606,31 @@ public class DataNode extends Configured
nsInfo.getVersion().equals(VersionInfo.getVersion());
boolean revisionMatch =
nsInfo.getRevision().equals(VersionInfo.getRevision());
-
if (revisionMatch && !versionMatch) {
throw new AssertionError("Invalid build. The revisions match" +
" but the NN version is " + nsInfo.getVersion() +
" and the DN version is " + VersionInfo.getVersion());
}
- if (relaxedVersionCheck) {
- if (versionMatch && !revisionMatch) {
- LOG.info("Permitting datanode revision " + VersionInfo.getRevision() +
- " to connect to namenode revision " + nsInfo.getRevision() +
- " because " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY +
- " is enabled");
- }
- return versionMatch;
+ if (noVersionCheck) {
+ LOG.info("Permitting datanode version '" + VersionInfo.getVersion() +
+ "' and revision '" + VersionInfo.getRevision() +
+ "' to connect to namenode version '" + nsInfo.getVersion() +
+ "' and revision '" + nsInfo.getRevision() + "' because " +
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY +
+ " is enabled");
+ return true;
} else {
- return revisionMatch;
+ if (relaxedVersionCheck) {
+ if (versionMatch && !revisionMatch) {
+ LOG.info("Permitting datanode revision " + VersionInfo.getRevision() +
+ " to connect to namenode revision " + nsInfo.getRevision() +
+ " because " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY +
+ " is enabled");
+ }
+ return versionMatch;
+ } else {
+ return revisionMatch;
+ }
}
}
@@ -633,12 +648,15 @@ public class DataNode extends Configured
}
}
if (!isPermittedVersion(nsInfo)) {
- String errorMsg = "Incompatible versions: namenode version " +
- nsInfo.getVersion() + " revision " + nsInfo.getRevision() +
- " datanode version " + VersionInfo.getVersion() + " revision " +
- VersionInfo.getRevision() + " and " +
- CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY +
- " is " + (relaxedVersionCheck ? "enabled" : "not enabled");
+ String errorMsg = "Shutting down. Incompatible version or revision." +
+ "DataNode version '" + VersionInfo.getVersion() +
+ "' and revision '" + VersionInfo.getRevision() +
+ "' and NameNode version '" + nsInfo.getVersion() +
+ "' and revision '" + nsInfo.getRevision() +
+ " and " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY +
+ " is " + (relaxedVersionCheck ? "enabled" : "not enabled") +
+ " and " + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY +
+ " is " + (noVersionCheck ? "enabled" : "not enabled");
LOG.fatal(errorMsg);
notifyNamenode(DatanodeProtocol.NOTIFY, errorMsg);
throw new IOException( errorMsg );
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1402741&r1=1402740&r2=1402741&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Sat Oct 27 04:30:41 2012
@@ -418,6 +418,11 @@ public class TaskTracker implements MRCo
*/
private boolean relaxedVersionCheck;
+ /**
+ * Whether the TT completely skips version check with the JT.
+ */
+ private boolean skipVersionCheck;
+
/*
* A list of commitTaskActions for whom commit response has been received
*/
@@ -1459,6 +1464,9 @@ public class TaskTracker implements MRCo
relaxedVersionCheck = conf.getBoolean(
CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY,
CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_DEFAULT);
+ skipVersionCheck = conf.getBoolean(
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY,
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_DEFAULT);
FILE_CACHE_SIZE = conf.getInt("mapred.tasktracker.file.cache.size", 2000);
maxMapSlots = conf.getInt(
"mapred.tasktracker.map.tasks.maximum", 2);
@@ -1623,7 +1631,7 @@ public class TaskTracker implements MRCo
private long getHeartbeatInterval(int numFinishedTasks) {
return (heartbeatInterval / (numFinishedTasks * oobHeartbeatDamper + 1));
}
-
+
/**
* @return true if this tasktracker is permitted to connect to
* the given jobtracker version
@@ -1637,16 +1645,26 @@ public class TaskTracker implements MRCo
" but the JT version is " + jtVersion +
" and the TT version is " + VersionInfo.getVersion());
}
- if (relaxedVersionCheck) {
- if (!buildVersionMatch && versionMatch) {
- LOG.info("Permitting tasktracker revision " + VersionInfo.getRevision() +
- " to connect to jobtracker " + jtBuildVersion + " because " +
- CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY +
- " is enabled");
- }
- return versionMatch;
+ if (skipVersionCheck) {
+ LOG.info("Permitting tasktracker version '" + VersionInfo.getVersion() +
+ "' and build '" + VersionInfo.getBuildVersion() +
+ "' to connect to jobtracker version '" + jtVersion +
+ "' and build '" + jtBuildVersion + "' because " +
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY +
+ " is enabled");
+ return true;
} else {
- return buildVersionMatch;
+ if (relaxedVersionCheck) {
+ if (!buildVersionMatch && versionMatch) {
+ LOG.info("Permitting tasktracker build " + VersionInfo.getBuildVersion() +
+ " to connect to jobtracker build " + jtBuildVersion + " because " +
+ CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY +
+ " is enabled");
+ }
+ return versionMatch;
+ } else {
+ return buildVersionMatch;
+ }
}
}
@@ -1689,11 +1707,15 @@ public class TaskTracker implements MRCo
String jtBuildVersion = jobClient.getBuildVersion();
String jtVersion = jobClient.getVIVersion();
if (!isPermittedVersion(jtBuildVersion, jtVersion)) {
- String msg = "Shutting down. Incompatible buildVersion." +
- "\nJobTracker's: " + jtBuildVersion +
- "\nTaskTracker's: "+ VersionInfo.getBuildVersion() +
+ String msg = "Shutting down. Incompatible version or build version." +
+ "TaskTracker version '" + VersionInfo.getVersion() +
+ "' and build '" + VersionInfo.getBuildVersion() +
+ "' and JobTracker version '" + jtVersion +
+ "' and build '" + jtBuildVersion +
" and " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY +
- " is " + (relaxedVersionCheck ? "enabled" : "not enabled");
+ " is " + (relaxedVersionCheck ? "enabled" : "not enabled") +
+ " and " + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY +
+ " is " + (skipVersionCheck ? "enabled" : "not enabled");
LOG.fatal(msg);
try {
jobClient.reportTaskTrackerError(taskTrackerName, null, msg);
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java?rev=1402741&r1=1402740&r2=1402741&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java Sat Oct 27 04:30:41 2012
@@ -116,4 +116,42 @@ public class TestDataNodeVersionCheck {
}
}
}
+
+ /**
+ * Test no DN version checking
+ */
+ @Test
+ public void testNoVersionCheck() throws IOException {
+ MiniDFSCluster cluster = null;
+ try {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY, true);
+ cluster = new MiniDFSCluster(conf, 1, true, null);
+
+ DataNode dn = cluster.getDataNodes().get(0);
+
+ final NamespaceInfo currInfo = new NamespaceInfo(0, 0, 0);
+ assertTrue(dn.isPermittedVersion(currInfo));
+
+ // Different revisions are permitted
+ NamespaceInfo infoDiffRev = new NamespaceInfo(0, 0, 0) {
+ @Override public String getRevision() { return "bogus"; }
+ };
+ assertTrue("Different revisions should be permitted",
+ dn.isPermittedVersion(infoDiffRev));
+
+ // Different versions are permitted
+ NamespaceInfo infoDiffVersion = new NamespaceInfo(0, 0, 0) {
+ @Override public String getVersion() { return "bogus"; }
+ @Override public String getRevision() { return "bogus"; }
+ };
+ assertTrue("Different versions should be permitted",
+ dn.isPermittedVersion(infoDiffVersion));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java?rev=1402741&r1=1402740&r2=1402741&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java Sat Oct 27 04:30:41 2012
@@ -88,4 +88,29 @@ public class TestTaskTrackerVersionCheck
}
}
}
+
+ /**
+ * Test no TT version checking
+ */
+ @Test
+ public void testNoVersionCheck() throws IOException {
+ MiniMRCluster mr = null;
+ try {
+ JobConf jtConf = new JobConf();
+ jtConf.setBoolean(
+ CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY, true);
+ mr = new MiniMRCluster(1, "file:///", 1, null, null, jtConf);
+ TaskTracker tt = mr.getTaskTrackerRunner(0).getTaskTracker();
+ String currFullVersion = VersionInfo.getBuildVersion();
+ String currVersion = VersionInfo.getVersion();
+
+ assertTrue(tt.isPermittedVersion(currFullVersion, currVersion));
+ assertTrue(tt.isPermittedVersion(currFullVersion+"x", currVersion+"x"));
+ assertTrue(tt.isPermittedVersion(currFullVersion+"x", currVersion));
+ } finally {
+ if (mr != null) {
+ mr.shutdown();
+ }
+ }
+ }
}