You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2012/12/05 00:26:37 UTC
svn commit: r1417238 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/core/org/apache/hadoop/conf/Configuration.java
src/test/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java
Author: szetszwo
Date: Tue Dec 4 23:26:36 2012
New Revision: 1417238
URL: http://svn.apache.org/viewvc?rev=1417238&view=rev
Log:
HADOOP-9115. Backport HADOOP-7082 and HDFS-1542: Configuration.writeXML should not hold lock while outputting and add a test for a deadlock writing Configuration to HDFS. Contributed by Jing Zhao
Added:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1417238&r1=1417237&r2=1417238&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Dec 4 23:26:36 2012
@@ -124,9 +124,6 @@ Release 1.2.0 - unreleased
HDFS-4122. Cleanup HDFS logs and reduce the size of logged messages.
(suresh)
- HDFS-4252. Improve confusing log message that prints exception when editlog
- read is completed. (Jing Zhao via suresh)
-
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
@@ -137,9 +134,6 @@ Release 1.2.0 - unreleased
BUG FIXES
- MAPREDUCE-4798. Updated TestJobHistoryServer test case for startup
- race conditions. (Sam Liu via eyang)
-
HADOOP-8460. Document proper setting of HADOOP_PID_DIR and
HADOOP_SECURE_DN_PID_DIR (bobby)
@@ -202,9 +196,6 @@ Release 1.2.0 - unreleased
HDFS-2751. Backport: Datanode may incorrectly drop OS cache behind reads
even for short reads. (Brandon Li via szetszwo)
- HDFS-3727. When using SPNEGO, NN should not try to log in using KSSL
- principal. (atm)
-
HADOOP-8613. AbstractDelegationTokenIdentifier#getUser() should set token
auth type. (daryn)
@@ -220,9 +211,6 @@ Release 1.2.0 - unreleased
HADOOP-8611. Allow fall-back to the shell-based implementation when
JNI-based users-group mapping fails (Robert Parker via bobby)
- MAPREDUCE-2374. "Text File Busy" errors launching MR tasks. (Andy Isaacson
- via atm)
-
HADOOP-4572. Can not access user logs - Jetty is not configured by default
to serve aliases/symlinks (ahmed via tucu)
@@ -322,19 +310,45 @@ Release 1.2.0 - unreleased
handles hostname starting with a numeric character. (Jing Zhao via
szetszwo)
- HDFS-4208. NameNode could be stuck in SafeMode due to never-created
- blocks. (Brandon Li via suresh)
-
HADOOP-9099. TestNetUtils fails if "UnknownHost" is resolved as a valid
hostname. (Ivan Mitic via szetszwo)
MAPREDUCE-4778. Fair scheduler event log is only written if directory
exists on HDFS. (Sandy Ryza via tomwhite)
+Release 1.1.2 - Unreleased
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ HDFS-4252. Improve confusing log message that prints exception when editlog
+ read is completed. (Jing Zhao via suresh)
+
+ BUG FIXES
+
+ MAPREDUCE-4798. Updated TestJobHistoryServer test case for startup
+ race conditions. (Sam Liu via eyang)
+
+ HDFS-3727. When using SPNEGO, NN should not try to log in using KSSL
+ principal. (atm)
+
+ MAPREDUCE-2374. "Text File Busy" errors launching MR tasks. (Andy Isaacson
+ via atm)
+
+ HDFS-4208. NameNode could be stuck in SafeMode due to never-created
+ blocks. (Brandon Li via suresh)
+
HADOOP-9111. Change some JUnit 3 tests to JUnit 4 so that @Ignore tests can
be run with ant 1.8.x. (Jing Zhao via szetszwo)
-Release 1.1.1 - Unreleased
+ HADOOP-9115. Backport HADOOP-7082 and HDFS-1542: Configuration.writeXML
+ should not hold lock while outputting and add a test for a deadlock writing
+ Configuration to HDFS. (Jing Zhao via szetszwo)
+
+Release 1.1.1 - 2012.11.18
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java?rev=1417238&r1=1417237&r2=1417238&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/conf/Configuration.java Tue Dec 4 23:26:36 2012
@@ -1269,51 +1269,65 @@ public class Configuration implements It
*
* @param out the writer to write to.
*/
- public synchronized void writeXml(Writer out) throws IOException {
- Properties properties = getProps();
+ public void writeXml(Writer out) throws IOException {
+ Document doc = asXmlDocument();
try {
- Document doc =
- DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
- Element conf = doc.createElement("configuration");
- doc.appendChild(conf);
- conf.appendChild(doc.createTextNode("\n"));
- for (Enumeration e = properties.keys(); e.hasMoreElements();) {
- String name = (String)e.nextElement();
- Object object = properties.get(name);
- String value = null;
- if (object instanceof String) {
- value = (String) object;
- }else {
- continue;
- }
- Element propNode = doc.createElement("property");
- conf.appendChild(propNode);
- if (updatingResource != null) {
- Comment commentNode = doc.createComment("Loaded from "
- + updatingResource.get(name));
- propNode.appendChild(commentNode);
- }
- Element nameNode = doc.createElement("name");
- nameNode.appendChild(doc.createTextNode(name));
- propNode.appendChild(nameNode);
-
- Element valueNode = doc.createElement("value");
- valueNode.appendChild(doc.createTextNode(value));
- propNode.appendChild(valueNode);
-
- conf.appendChild(doc.createTextNode("\n"));
- }
-
DOMSource source = new DOMSource(doc);
StreamResult result = new StreamResult(out);
TransformerFactory transFactory = TransformerFactory.newInstance();
Transformer transformer = transFactory.newTransformer();
+
+ // Important to not hold Configuration log while writing result, since
+ // 'out' may be an HDFS stream which needs to lock this configuration
+ // from another thread.
transformer.transform(source, result);
} catch (TransformerException te) {
throw new IOException(te);
+ }
+ }
+
+ /**
+ * Return the XML DOM corresponding to this Configuration.
+ */
+ private synchronized Document asXmlDocument() throws IOException {
+ Document doc;
+ Properties properties = getProps();
+ try {
+ doc =
+ DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
} catch (ParserConfigurationException pe) {
throw new IOException(pe);
}
+ Element conf = doc.createElement("configuration");
+ doc.appendChild(conf);
+ conf.appendChild(doc.createTextNode("\n"));
+ for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
+ String name = (String) e.nextElement();
+ Object object = properties.get(name);
+ String value = null;
+ if (object instanceof String) {
+ value = (String) object;
+ } else {
+ continue;
+ }
+ Element propNode = doc.createElement("property");
+ conf.appendChild(propNode);
+ if (updatingResource != null) {
+ Comment commentNode = doc.createComment("Loaded from "
+ + updatingResource.get(name));
+ propNode.appendChild(commentNode);
+ }
+ Element nameNode = doc.createElement("name");
+ nameNode.appendChild(doc.createTextNode(name));
+ propNode.appendChild(nameNode);
+
+ Element valueNode = doc.createElement("value");
+ valueNode.appendChild(doc.createTextNode(value));
+ propNode.appendChild(valueNode);
+
+ conf.appendChild(doc.createTextNode("\n"));
+ }
+ return doc;
}
/**
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java?rev=1417238&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java (added)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestWriteConfigurationToDFS.java Tue Dec 4 23:26:36 2012
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import java.io.OutputStream;
+import org.junit.Test;
+
+/**
+ * Regression test for HDFS-1542, a deadlock between the main thread
+ * and the DFSOutputStream.DataStreamer thread caused because
+ * Configuration.writeXML holds a lock on itself while writing to DFS.
+ */
+public class TestWriteConfigurationToDFS {
+ @Test(timeout=60000)
+ public void testWriteConf() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096);
+ System.out.println("Setting conf in: " + System.identityHashCode(conf));
+
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null, null);
+ cluster.waitActive();
+ try {
+ FileSystem fs = cluster.getFileSystem();
+ Path filePath = new Path("/testWriteConf.xml");
+ OutputStream os = fs.create(filePath);
+ StringBuilder longString = new StringBuilder();
+ for (int i = 0; i < 100000; i++) {
+ longString.append("hello");
+ } // 500KB
+ conf.set("foobar", longString.toString());
+ conf.writeXml(os);
+ os.close();
+ } finally {
+ cluster.shutdown();
+ }
+ }
+}
\ No newline at end of file