You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/08/01 22:25:19 UTC
svn commit: r1509417 - in /hadoop/common/branches/branch-1-win:
CHANGES.branch-1-win.txt src/core/org/apache/hadoop/conf/Configuration.java
src/test/org/apache/hadoop/conf/TestConfiguration.java
Author: cnauroth
Date: Thu Aug 1 20:25:19 2013
New Revision: 1509417
URL: http://svn.apache.org/r1509417
Log:
HADOOP-9801. Configuration#writeXml uses platform defaulting encoding, which may mishandle multi-byte characters. Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/conf/TestConfiguration.java
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1509417&r1=1509416&r2=1509417&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Thu Aug 1 20:25:19 2013
@@ -478,3 +478,6 @@ Branch-hadoop-1-win (branched from branc
HADOOP-9768. chown and chgrp reject users and groups with spaces on platforms
where spaces are otherwise acceptable. (cnauroth)
+
+ HADOOP-9801. Configuration#writeXml uses platform defaulting encoding, which
+ may mishandle multi-byte characters. (cnauroth)
Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/conf/Configuration.java?rev=1509417&r1=1509416&r2=1509417&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/conf/Configuration.java Thu Aug 1 20:25:19 2013
@@ -1267,12 +1267,12 @@ public class Configuration implements It
/**
* Write out the non-default properties in this configuration to the given
- * {@link OutputStream}.
+ * {@link OutputStream} using UTF-8 encoding.
*
* @param out the output stream to write to.
*/
public void writeXml(OutputStream out) throws IOException {
- writeXml(new OutputStreamWriter(out));
+ writeXml(new OutputStreamWriter(out, "UTF-8"));
}
/**
Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/conf/TestConfiguration.java?rev=1509417&r1=1509416&r2=1509417&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/conf/TestConfiguration.java (original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/conf/TestConfiguration.java Thu Aug 1 20:25:19 2013
@@ -24,7 +24,9 @@ import java.io.IOException;
import java.io.DataInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
+import java.io.FileOutputStream;
import java.io.DataOutputStream;
+import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
@@ -34,6 +36,7 @@ import java.util.Random;
import junit.framework.TestCase;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.codehaus.jackson.map.ObjectMapper;
@@ -42,6 +45,10 @@ public class TestConfiguration extends T
private Configuration conf;
final static String CONFIG = new File("./test-config.xml").getAbsolutePath();
final static String CONFIG2 = new File("./test-config2.xml").getAbsolutePath();
+ private static final String CONFIG_MULTI_BYTE = new File(
+ "./test-config-multi-byte-TestConfiguration.xml").getAbsolutePath();
+ private static final String CONFIG_MULTI_BYTE_SAVED = new File(
+ "./test-config-multi-byte-saved-TestConfiguration.xml").getAbsolutePath();
final static Random RAN = new Random();
@Override
@@ -55,6 +62,8 @@ public class TestConfiguration extends T
super.tearDown();
new File(CONFIG).delete();
new File(CONFIG2).delete();
+ new File(CONFIG_MULTI_BYTE).delete();
+ new File(CONFIG_MULTI_BYTE_SAVED).delete();
}
private void startConfig() throws IOException{
@@ -71,6 +80,41 @@ public class TestConfiguration extends T
out.write("<xi:include href=\"" + filename + "\" xmlns:xi=\"http://www.w3.org/2001/XInclude\" />\n ");
}
+ /**
+ * Tests use of multi-byte characters in property names and values. This test
+ * round-trips multi-byte string literals through saving and loading of config
+ * and asserts that the same values were read.
+ */
+ public void testMultiByteCharacters() throws IOException {
+ String priorDefaultEncoding = System.getProperty("file.encoding");
+ try {
+ System.setProperty("file.encoding", "US-ASCII");
+ String name = "multi_byte_\u611b_name";
+ String value = "multi_byte_\u0641_value";
+ out = new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(CONFIG_MULTI_BYTE), "UTF-8"));
+ startConfig();
+ declareProperty(name, value, value);
+ endConfig();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(CONFIG_MULTI_BYTE));
+ assertEquals(value, conf.get(name));
+ FileOutputStream fos = new FileOutputStream(CONFIG_MULTI_BYTE_SAVED);
+ try {
+ conf.writeXml(fos);
+ } finally {
+ IOUtils.closeStream(fos);
+ }
+
+ conf = new Configuration(false);
+ conf.addResource(new Path(CONFIG_MULTI_BYTE_SAVED));
+ assertEquals(value, conf.get(name));
+ } finally {
+ System.setProperty("file.encoding", priorDefaultEncoding);
+ }
+ }
+
public void testVariableSubstitution() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();