You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by pa...@apache.org on 2018/01/14 10:40:42 UTC
[1/2] commons-io git commit: IO-557: UnsupportedEncodingException
when opening an ISO-8859-1 XML stream with Turkish as the default locale
(closes #51)
Repository: commons-io
Updated Branches:
refs/heads/master d3fc1a287 -> ed15b0998
IO-557: UnsupportedEncodingException when opening an ISO-8859-1 XML stream with Turkish as the default locale (closes #51)
Perform locale independent upper case conversions.
To handle properly lower cased character encoding name in XML prolog
with any default system locale, notably Turkish.
Project: http://git-wip-us.apache.org/repos/asf/commons-io/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-io/commit/d19259a7
Tree: http://git-wip-us.apache.org/repos/asf/commons-io/tree/d19259a7
Diff: http://git-wip-us.apache.org/repos/asf/commons-io/diff/d19259a7
Branch: refs/heads/master
Commit: d19259a7774e23ebdaac4af5ee4def8602fe2969
Parents: d3fc1a2
Author: luccioman <lu...@users.noreply.github.com>
Authored: Mon Dec 18 09:37:06 2017 +0100
Committer: pascalschumacher <pa...@gmx.net>
Committed: Sun Jan 14 11:39:38 2018 +0100
----------------------------------------------------------------------
.../org/apache/commons/io/ByteOrderMark.java | 3 ++-
.../apache/commons/io/input/XmlStreamReader.java | 4 ++--
.../commons/io/output/XmlStreamWriter.java | 3 ++-
.../commons/io/input/XmlStreamReaderTest.java | 19 +++++++++++++++++++
.../io/input/compatibility/XmlStreamReader.java | 5 +++--
.../commons/io/output/XmlStreamWriterTest.java | 16 ++++++++++++++--
6 files changed, 42 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-io/blob/d19259a7/src/main/java/org/apache/commons/io/ByteOrderMark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/io/ByteOrderMark.java b/src/main/java/org/apache/commons/io/ByteOrderMark.java
index 109c402..6840e7e 100644
--- a/src/main/java/org/apache/commons/io/ByteOrderMark.java
+++ b/src/main/java/org/apache/commons/io/ByteOrderMark.java
@@ -17,6 +17,7 @@
package org.apache.commons.io;
import java.io.Serializable;
+import java.util.Locale;
/**
* Byte Order Mark (BOM) representation - see {@link org.apache.commons.io.input.BOMInputStream}.
@@ -182,7 +183,7 @@ public class ByteOrderMark implements Serializable {
builder.append(",");
}
builder.append("0x");
- builder.append(Integer.toHexString(0xFF & bytes[i]).toUpperCase());
+ builder.append(Integer.toHexString(0xFF & bytes[i]).toUpperCase(Locale.ROOT));
}
builder.append(']');
return builder.toString();
http://git-wip-us.apache.org/repos/asf/commons-io/blob/d19259a7/src/main/java/org/apache/commons/io/input/XmlStreamReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/io/input/XmlStreamReader.java b/src/main/java/org/apache/commons/io/input/XmlStreamReader.java
index d1d24b3..f729b54 100644
--- a/src/main/java/org/apache/commons/io/input/XmlStreamReader.java
+++ b/src/main/java/org/apache/commons/io/input/XmlStreamReader.java
@@ -683,7 +683,7 @@ public class XmlStreamReader extends Reader {
final String postMime = httpContentType.substring(i + 1);
final Matcher m = CHARSET_PATTERN.matcher(postMime);
encoding = m.find() ? m.group(1) : null;
- encoding = encoding != null ? encoding.toUpperCase(Locale.US) : null;
+ encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
}
}
return encoding;
@@ -741,7 +741,7 @@ public class XmlStreamReader extends Reader {
}
final Matcher m = ENCODING_PATTERN.matcher(prolog);
if (m.find()) {
- encoding = m.group(1).toUpperCase();
+ encoding = m.group(1).toUpperCase(Locale.ROOT);
encoding = encoding.substring(1, encoding.length() - 1);
}
}
http://git-wip-us.apache.org/repos/asf/commons-io/blob/d19259a7/src/main/java/org/apache/commons/io/output/XmlStreamWriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/io/output/XmlStreamWriter.java b/src/main/java/org/apache/commons/io/output/XmlStreamWriter.java
index d352ad0..28fa6ac 100644
--- a/src/main/java/org/apache/commons/io/output/XmlStreamWriter.java
+++ b/src/main/java/org/apache/commons/io/output/XmlStreamWriter.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.io.Writer;
+import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -168,7 +169,7 @@ public class XmlStreamWriter extends Writer {
final Matcher m = ENCODING_PATTERN.matcher(xmlProlog.substring(0,
xmlPrologEnd));
if (m.find()) {
- encoding = m.group(1).toUpperCase();
+ encoding = m.group(1).toUpperCase(Locale.ROOT);
encoding = encoding.substring(1, encoding.length() - 1);
} else {
// no encoding found in XML prolog: using default
http://git-wip-us.apache.org/repos/asf/commons-io/blob/d19259a7/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java b/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
index 183e1a4..a968c56 100644
--- a/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
+++ b/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
@@ -276,6 +276,25 @@ public class XmlStreamReaderTest {
_testHttpLenient("text/html;charset=UTF-16BE", "no-bom", "US-ASCII", "UTF-8", "UTF-8");
_testHttpLenient("text/html;charset=UTF-32BE", "no-bom", "US-ASCII", "UTF-8", "UTF-8");
}
+
+ /**
+ * Check lower case encoding names are properly handled. Should be successfull
+ * with any system default locale, notably with Turkish language
+ * (-Duser.language=tr JVM parameter), which has specific rules to convert dotted and dottless
+ * i character.
+ */
+ @Test
+ public void testLowerCaseEncoding() throws Exception {
+ final String[] encodings = { "iso8859-1", "us-ascii", "utf-8" };
+ for (final String encoding : encodings) {
+ final String xml = getXML("no-bom", XML3, encoding, encoding);
+ try (final ByteArrayInputStream is = new ByteArrayInputStream(xml.getBytes(encoding));
+ final XmlStreamReader xmlReader = new XmlStreamReader(is);) {
+ assertTrue("Check encoding : " + encoding, encoding.equalsIgnoreCase(xmlReader.getEncoding()));
+ assertEquals("Check content", xml, IOUtils.toString(xmlReader));
+ }
+ }
+ }
@Test
public void testRawContent() throws Exception {
http://git-wip-us.apache.org/repos/asf/commons-io/blob/d19259a7/src/test/java/org/apache/commons/io/input/compatibility/XmlStreamReader.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/io/input/compatibility/XmlStreamReader.java b/src/test/java/org/apache/commons/io/input/compatibility/XmlStreamReader.java
index be5b121..5442315 100644
--- a/src/test/java/org/apache/commons/io/input/compatibility/XmlStreamReader.java
+++ b/src/test/java/org/apache/commons/io/input/compatibility/XmlStreamReader.java
@@ -29,6 +29,7 @@ import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.text.MessageFormat;
+import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -590,7 +591,7 @@ public class XmlStreamReader extends Reader {
final String postMime = httpContentType.substring(i + 1);
final Matcher m = CHARSET_PATTERN.matcher(postMime);
encoding = m.find() ? m.group(1) : null;
- encoding = encoding != null ? encoding.toUpperCase() : null;
+ encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
}
}
return encoding;
@@ -699,7 +700,7 @@ public class XmlStreamReader extends Reader {
}
final Matcher m = ENCODING_PATTERN.matcher(prolog);
if (m.find()) {
- encoding = m.group(1).toUpperCase();
+ encoding = m.group(1).toUpperCase(Locale.ROOT);
encoding = encoding.substring(1, encoding.length() - 1);
}
}
http://git-wip-us.apache.org/repos/asf/commons-io/blob/d19259a7/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java b/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
index 0ee2507..b8f578c 100644
--- a/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
+++ b/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
@@ -16,7 +16,6 @@
*/
package org.apache.commons.io.output;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
@@ -56,7 +55,7 @@ public class XmlStreamWriterTest {
writer.write(xml);
writer.close();
final byte[] xmlContent = out.toByteArray();
- assertEquals(encoding, writer.getEncoding());
+ assertTrue(encoding.equalsIgnoreCase(writer.getEncoding()));
assertTrue(Arrays.equals(xml.getBytes(encoding), xmlContent));
}
@@ -102,6 +101,19 @@ public class XmlStreamWriterTest {
checkXmlWriter(TEXT_UNICODE, null, "UTF-16BE");
checkXmlWriter(TEXT_UNICODE, null, "ISO-8859-1");
}
+
+ /**
+ * Check lower case encoding names are properly handled. Should be successfull
+ * with any system default locale, notably with Turkish language
+ * (-Duser.language=tr JVM parameter), which has specific rules to convert
+ * dotted and dottless i character.
+ */
+ @Test
+ public void testLowerCaseEncoding() throws IOException {
+ checkXmlWriter(TEXT_UNICODE, "utf-8");
+ checkXmlWriter(TEXT_LATIN1, "iso-8859-1");
+ checkXmlWriter(TEXT_LATIN7, "iso-8859-7");
+ }
@Test
public void testUTF8Encoding() throws IOException {
[2/2] commons-io git commit: IO-557: UnsupportedEncodingException
when opening an ISO-8859-1 XML stream with Turkish as the default locale
Posted by pa...@apache.org.
IO-557: UnsupportedEncodingException when opening an ISO-8859-1 XML stream with Turkish as the default locale
improve tests by copying parts of the SystemDefaults rule from commons-lang
add changes.xml entry
Project: http://git-wip-us.apache.org/repos/asf/commons-io/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-io/commit/ed15b099
Tree: http://git-wip-us.apache.org/repos/asf/commons-io/tree/ed15b099
Diff: http://git-wip-us.apache.org/repos/asf/commons-io/diff/ed15b099
Branch: refs/heads/master
Commit: ed15b09985a073d0b17769bb9a28d3e41419a43b
Parents: d19259a
Author: pascalschumacher <pa...@gmx.net>
Authored: Sun Jan 14 11:38:57 2018 +0100
Committer: pascalschumacher <pa...@gmx.net>
Committed: Sun Jan 14 11:40:34 2018 +0100
----------------------------------------------------------------------
src/changes/changes.xml | 3 +
.../commons/io/input/XmlStreamReaderTest.java | 18 +++--
.../commons/io/output/XmlStreamWriterTest.java | 18 +++--
.../commons/io/testtools/SystemDefaults.java | 36 +++++++++
.../io/testtools/SystemDefaultsSwitch.java | 79 ++++++++++++++++++++
5 files changed, 138 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-io/blob/ed15b099/src/changes/changes.xml
----------------------------------------------------------------------
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index b308d2f..a70b4ee 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -50,6 +50,9 @@ The <action> type attribute can be add,update,fix,remove.
<action issue="IO-553" dev="ggregory" type="add">
Add org.apache.commons.io.FilenameUtils.isIllegalWindowsFileName(char).
</action>
+ <action issue="IO-557" dev="pschumacher" type="fix" due-to="luccioman">
+ Perform locale independent upper case conversions.
+ </action>
</release>
<release version="2.6" date="2017-10-15" description="Java 7 required, Java 9 supported.">
http://git-wip-us.apache.org/repos/asf/commons-io/blob/ed15b099/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java b/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
index a968c56..cdb8020 100644
--- a/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
+++ b/src/test/java/org/apache/commons/io/input/XmlStreamReaderTest.java
@@ -31,6 +31,9 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.testtools.SystemDefaults;
+import org.apache.commons.io.testtools.SystemDefaultsSwitch;
+import org.junit.Rule;
import org.junit.Test;
public class XmlStreamReaderTest {
@@ -40,6 +43,9 @@ public class XmlStreamReaderTest {
private static final String XML2 = "xml-prolog";
private static final String XML1 = "xml";
+ @Rule
+ public SystemDefaultsSwitch locale = new SystemDefaultsSwitch();
+
protected void _testRawNoBomValid(final String encoding) throws Exception {
InputStream is = getXmlStream("no-bom", XML1, encoding, encoding);
XmlStreamReader xmlReader = new XmlStreamReader(is, false);
@@ -276,15 +282,11 @@ public class XmlStreamReaderTest {
_testHttpLenient("text/html;charset=UTF-16BE", "no-bom", "US-ASCII", "UTF-8", "UTF-8");
_testHttpLenient("text/html;charset=UTF-32BE", "no-bom", "US-ASCII", "UTF-8", "UTF-8");
}
-
- /**
- * Check lower case encoding names are properly handled. Should be successfull
- * with any system default locale, notably with Turkish language
- * (-Duser.language=tr JVM parameter), which has specific rules to convert dotted and dottless
- * i character.
- */
+
+ // Turkish language has specific rules to convert dotted and dottless i character.
@Test
- public void testLowerCaseEncoding() throws Exception {
+ @SystemDefaults(locale="tr")
+ public void testLowerCaseEncodingWithTurkishLocale_IO_557() throws Exception {
final String[] encodings = { "iso8859-1", "us-ascii", "utf-8" };
for (final String encoding : encodings) {
final String xml = getXML("no-bom", XML3, encoding, encoding);
http://git-wip-us.apache.org/repos/asf/commons-io/blob/ed15b099/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java b/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
index b8f578c..7d3cd13 100644
--- a/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
+++ b/src/test/java/org/apache/commons/io/output/XmlStreamWriterTest.java
@@ -22,6 +22,9 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.commons.io.testtools.SystemDefaults;
+import org.apache.commons.io.testtools.SystemDefaultsSwitch;
+import org.junit.Rule;
import org.junit.Test;
/**
@@ -40,6 +43,9 @@ public class XmlStreamWriterTest {
private static final String TEXT_UNICODE = TEXT_LATIN1 + ", " + TEXT_LATIN7
+ ", " + TEXT_LATIN15 + ", " + TEXT_EUC_JP;
+ @Rule
+ public SystemDefaultsSwitch locale = new SystemDefaultsSwitch();
+
private static String createXmlContent(final String text, final String encoding) {
String xmlDecl = "<?xml version=\"1.0\"?>";
if (encoding != null) {
@@ -101,15 +107,11 @@ public class XmlStreamWriterTest {
checkXmlWriter(TEXT_UNICODE, null, "UTF-16BE");
checkXmlWriter(TEXT_UNICODE, null, "ISO-8859-1");
}
-
- /**
- * Check lower case encoding names are properly handled. Should be successfull
- * with any system default locale, notably with Turkish language
- * (-Duser.language=tr JVM parameter), which has specific rules to convert
- * dotted and dottless i character.
- */
+
+ // Turkish language has specific rules to convert dotted and dottless i character.
@Test
- public void testLowerCaseEncoding() throws IOException {
+ @SystemDefaults(locale="tr")
+ public void testLowerCaseEncoding_IO_557() throws IOException {
checkXmlWriter(TEXT_UNICODE, "utf-8");
checkXmlWriter(TEXT_LATIN1, "iso-8859-1");
checkXmlWriter(TEXT_LATIN7, "iso-8859-7");
http://git-wip-us.apache.org/repos/asf/commons-io/blob/ed15b099/src/test/java/org/apache/commons/io/testtools/SystemDefaults.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/io/testtools/SystemDefaults.java b/src/test/java/org/apache/commons/io/testtools/SystemDefaults.java
new file mode 100644
index 0000000..ea2b76d
--- /dev/null
+++ b/src/test/java/org/apache/commons/io/testtools/SystemDefaults.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.io.testtools;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation used with {@link SystemDefaults} that specifies the
+ * system default Locale to be used in a test method.
+ */
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SystemDefaults {
+ /**
+ * The name of the Locale to be used while running a test method
+ */
+ String locale() default "";
+}
http://git-wip-us.apache.org/repos/asf/commons-io/blob/ed15b099/src/test/java/org/apache/commons/io/testtools/SystemDefaultsSwitch.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/io/testtools/SystemDefaultsSwitch.java b/src/test/java/org/apache/commons/io/testtools/SystemDefaultsSwitch.java
new file mode 100644
index 0000000..ae137df
--- /dev/null
+++ b/src/test/java/org/apache/commons/io/testtools/SystemDefaultsSwitch.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.io.testtools;
+
+import java.util.Locale;
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+/**
+ * Test Rule used with {@link SystemDefaults} annotation that sets and restores the system default Locale.
+ *
+ * <p>
+ * Set up tests to use alternate system default Locale by creating an instance of this rule
+ * and annotating the test method with {@link SystemDefaults}
+ * </p>
+ *
+ * <pre>
+ * public class SystemDefaultsDependentTest {
+ *
+ * {@literal @}Rule
+ * public SystemDefaultsSwitch locale = new SystemDefaultsSwitch();
+ *
+ * {@literal @}Test
+ * {@literal @}SystemDefaults(local="zh_CN")
+ * public void testWithSimplifiedChinaDefaultLocale() {
+ * // Locale.getDefault() will return Locale.CHINA until the end of this test method
+ * }
+ * }
+ * </pre>
+ */
+public class SystemDefaultsSwitch implements TestRule {
+
+ @Override
+ public Statement apply(final Statement stmt, final Description description) {
+ final SystemDefaults defaults = description.getAnnotation(SystemDefaults.class);
+ if (defaults == null) {
+ return stmt;
+ }
+ return applyLocale(defaults, stmt);
+ }
+
+ private Statement applyLocale(final SystemDefaults defaults, final Statement stmt) {
+ if (defaults.locale().isEmpty()) {
+ return stmt;
+ }
+
+ final Locale newLocale = Locale.forLanguageTag(defaults.locale());
+
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ final Locale save = Locale.getDefault();
+ try {
+ Locale.setDefault(newLocale);
+ stmt.evaluate();
+ } finally {
+ Locale.setDefault(save);
+ }
+ }
+ };
+ }
+
+}