You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2016/01/19 03:34:19 UTC
svn commit: r1725407 - in /commons/proper/csv/trunk/src: changes/changes.xml
main/java/org/apache/commons/csv/CSVFormat.java
main/java/org/apache/commons/csv/CSVPrinter.java
test/java/org/apache/commons/csv/CSVPrinterTest.java
Author: ggregory
Date: Tue Jan 19 02:34:19 2016
New Revision: 1725407
URL: http://svn.apache.org/viewvc?rev=1725407&view=rev
Log:
[CSV-168] CsvFormat.nullString should not be escaped. [CSV-170]
CSVFormat.MYSQL nullString should be "\N".
Modified:
commons/proper/csv/trunk/src/changes/changes.xml
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVFormat.java
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVPrinter.java
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
Modified: commons/proper/csv/trunk/src/changes/changes.xml
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/changes/changes.xml?rev=1725407&r1=1725406&r2=1725407&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/changes/changes.xml (original)
+++ commons/proper/csv/trunk/src/changes/changes.xml Tue Jan 19 02:34:19 2016
@@ -42,6 +42,8 @@
<action issue="CSV-153" type="update" dev="britter" due-to="Wren">CSVPrinter doesn't skip creation of header record if skipHeaderRecord is set to true</action>
<action issue="CSV-159" type="add" dev="ggregory" due-to="Yamil Medina">Add IgnoreCase option for accessing header names</action>
<action issue="CSV-169" type="add" dev="ggregory" due-to="Gary Gregory">The null string should be case-sensitive when reading records</action>
+ <action issue="CSV-168" type="fix" dev="ggregory" due-to="Gary Gregory, cornel creanga">CsvFormat.nullString should not be escaped</action>
+ <action issue="CSV-170" type="fix" dev="ggregory" due-to="Gary Gregory, cornel creanga">CSVFormat.MYSQL nullString should be "\N"</action>
</release>
<release version="1.2" date="2015-08-24" description="Feature and bug fix release">
<action issue="CSV-145" type="fix" dev="ggregory" due-to="Frank Ulbricht">CSVFormat.with* methods clear the header comments</action>
Modified: commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVFormat.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVFormat.java?rev=1725407&r1=1725406&r2=1725407&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVFormat.java (original)
+++ commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVFormat.java Tue Jan 19 02:34:19 2016
@@ -296,7 +296,7 @@ public final class CSVFormat implements
*
* <p>
* This is a tab-delimited format with a LF character as the line separator. Values are not quoted and special
- * characters are escaped with '\'.
+ * characters are escaped with {@code '\'}. The default NULL string is {@code "\\N"}.
* </p>
*
* <p>
@@ -308,6 +308,7 @@ public final class CSVFormat implements
* <li>withRecordSeparator('\n')</li>
* <li>withIgnoreEmptyLines(false)</li>
* <li>withEscape('\\')</li>
+ * <li>withNullString("\\N")</li>
* </ul>
*
* @see Predefined#MySQL
@@ -315,7 +316,7 @@ public final class CSVFormat implements
* http://dev.mysql.com/doc/refman/5.1/en/load-data.html</a>
*/
public static final CSVFormat MYSQL = DEFAULT.withDelimiter(TAB).withEscape(BACKSLASH).withIgnoreEmptyLines(false)
- .withQuote(null).withRecordSeparator(LF);
+ .withQuote(null).withRecordSeparator(LF).withNullString("\\N");
/**
* Returns true if the given character is a line break character.
Modified: commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVPrinter.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVPrinter.java?rev=1725407&r1=1725406&r2=1725407&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVPrinter.java (original)
+++ commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVPrinter.java Tue Jan 19 02:34:19 2016
@@ -136,7 +136,9 @@ public final class CSVPrinter implements
if (!newRecord) {
out.append(format.getDelimiter());
}
- if (format.isQuoteCharacterSet()) {
+ if (object == null) {
+ out.append(value);
+ } else if (format.isQuoteCharacterSet()) {
// the original object is needed so can check for Number
printAndQuote(object, value, offset, len);
} else if (format.isEscapeCharacterSet()) {
Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVPrinterTest.java?rev=1725407&r1=1725406&r2=1725407&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVPrinterTest.java (original)
+++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVPrinterTest.java Tue Jan 19 02:34:19 2016
@@ -18,6 +18,7 @@
package org.apache.commons.csv;
import static org.apache.commons.csv.Constants.CR;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -35,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
+import org.apache.commons.lang3.ObjectUtils;
import org.junit.Test;
/**
@@ -44,6 +46,8 @@ import org.junit.Test;
*/
public class CSVPrinterTest {
+ private static final int ITERATIONS_FOR_RANDOM_TEST = 50000;
+
private final String recordSeparator = CSVFormat.DEFAULT.getRecordSeparator();
private static String printable(final String s) {
@@ -65,21 +69,14 @@ public class CSVPrinterTest {
final int nLines = r.nextInt(4) + 1;
final int nCol = r.nextInt(3) + 1;
// nLines=1;nCol=2;
- final String[][] lines = new String[nLines][];
- for (int i = 0; i < nLines; i++) {
- final String[] line = new String[nCol];
- lines[i] = line;
- for (int j = 0; j < nCol; j++) {
- line[j] = randStr();
- }
- }
+ final String[][] lines = generateLines(nLines, nCol);
final StringWriter sw = new StringWriter();
final CSVPrinter printer = new CSVPrinter(sw, format);
for (int i = 0; i < nLines; i++) {
// for (int j=0; j<lines[i].length; j++) System.out.println("### VALUE=:" + printable(lines[i][j]));
- printer.printRecord((Object[])lines[i]);
+ printer.printRecord((Object[]) lines[i]);
}
printer.flush();
@@ -90,10 +87,26 @@ public class CSVPrinterTest {
final CSVParser parser = CSVParser.parse(result, format);
final List<CSVRecord> parseResult = parser.getRecords();
- Utils.compare("Printer output :" + printable(result), lines, parseResult);
+ String[][] expected = lines.clone();
+ for (int i = 0; i < expected.length; i++) {
+ expected[i] = expectNulls(expected[i], format);
+ }
+ Utils.compare("Printer output :" + printable(result), expected, parseResult);
parser.close();
}
+ private String[][] generateLines(final int nLines, final int nCol) {
+ final String[][] lines = new String[nLines][];
+ for (int i = 0; i < nLines; i++) {
+ final String[] line = new String[nCol];
+ lines[i] = line;
+ for (int j = 0; j < nCol; j++) {
+ line[j] = randStr();
+ }
+ }
+ return lines;
+ }
+
private void doRandom(final CSVFormat format, final int iter) throws Exception {
for (int i = 0; i < iter; i++) {
doOneRandom(format);
@@ -111,37 +124,37 @@ public class CSVPrinterTest {
char ch;
final int what = r.nextInt(20);
switch (what) {
- case 0:
- ch = '\r';
- break;
- case 1:
- ch = '\n';
- break;
- case 2:
- ch = '\t';
- break;
- case 3:
- ch = '\f';
- break;
- case 4:
- ch = ' ';
- break;
- case 5:
- ch = ',';
- break;
- case 6:
- ch = '"';
- break;
- case 7:
- ch = '\'';
- break;
- case 8:
- ch = '\\';
- break;
- default:
- ch = (char) r.nextInt(300);
- break;
- // default: ch = 'a'; break;
+ case 0:
+ ch = '\r';
+ break;
+ case 1:
+ ch = '\n';
+ break;
+ case 2:
+ ch = '\t';
+ break;
+ case 3:
+ ch = '\f';
+ break;
+ case 4:
+ ch = ' ';
+ break;
+ case 5:
+ ch = ',';
+ break;
+ case 6:
+ ch = '"';
+ break;
+ case 7:
+ ch = '\'';
+ break;
+ case 8:
+ ch = '\\';
+ break;
+ default:
+ ch = (char) r.nextInt(300);
+ break;
+ // default: ch = 'a'; break;
}
buf[i] = ch;
}
@@ -189,8 +202,8 @@ public class CSVPrinterTest {
public void testExcelPrintAllIterableOfLists() throws IOException {
final StringWriter sw = new StringWriter();
final CSVPrinter printer = new CSVPrinter(sw, CSVFormat.EXCEL);
- printer.printRecords(Arrays.asList(new List[] { Arrays.asList("r1c1", "r1c2"),
- Arrays.asList("r2c1", "r2c2") }));
+ printer.printRecords(
+ Arrays.asList(new List[] { Arrays.asList("r1c1", "r1c2"), Arrays.asList("r2c1", "r2c2") }));
assertEquals("r1c1,r1c2" + recordSeparator + "r2c1,r2c2" + recordSeparator, sw.toString());
printer.close();
}
@@ -302,6 +315,131 @@ public class CSVPrinterTest {
}
@Test
+ public void testMySqlNullStringDefault() throws IOException {
+ assertEquals("\\N", CSVFormat.MYSQL.getNullString());
+ }
+
+ @Test
+ public void testMySqlNullOutput() throws IOException {
+ Object[] s = new String[] { "NULL", null };
+ CSVFormat format = CSVFormat.MYSQL.withQuote('"').withNullString("NULL").withQuoteMode(QuoteMode.NON_NUMERIC);
+ StringWriter writer = new StringWriter();
+ CSVPrinter printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ String expected = "\"NULL\"\tNULL\n";
+ assertEquals(expected, writer.toString());
+ String[] record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(new Object[2], record0);
+
+ s = new String[] { "\\N", null };
+ format = CSVFormat.MYSQL.withNullString("\\N");
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "\\\\N\t\\N\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+
+ s = new String[] { "\\N", "A" };
+ format = CSVFormat.MYSQL.withNullString("\\N");
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "\\\\N\tA\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+
+ s = new String[] { "\n", "A" };
+ format = CSVFormat.MYSQL.withNullString("\\N");
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "\\n\tA\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+
+ s = new String[] { "", null };
+ format = CSVFormat.MYSQL.withNullString("NULL");
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "\tNULL\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+
+ s = new String[] { "", null };
+ format = CSVFormat.MYSQL;
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "\t\\N\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+
+ s = new String[] { "\\N", "", "\u000e,\\\r" };
+ format = CSVFormat.MYSQL;
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "\\\\N\t\t\u000e,\\\\\\r\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+
+ s = new String[] { "NULL", "\\\r" };
+ format = CSVFormat.MYSQL;
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "NULL\t\\\\\\r\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+
+ s = new String[] { "\\\r" };
+ format = CSVFormat.MYSQL;
+ writer = new StringWriter();
+ printer = new CSVPrinter(writer, format);
+ printer.printRecord(s);
+ printer.close();
+ expected = "\\\\\\r\n";
+ assertEquals(expected, writer.toString());
+ record0 = toFirstRecordValues(expected, format);
+ assertArrayEquals(expectNulls(s, format), record0);
+ }
+
+ /**
+ * Converts an input CSV array into expected output values WRT NULLs. NULL strings are converted to null values
+ * because the parser will convert these strings to null.
+ */
+ private <T> T[] expectNulls(T[] original, CSVFormat csvFormat) {
+ T[] fixed = original.clone();
+ for (int i = 0; i < fixed.length; i++) {
+ if (ObjectUtils.equals(csvFormat.getNullString(), fixed[i])) {
+ fixed[i] = null;
+ }
+ }
+ return fixed;
+ }
+
+ private String[] toFirstRecordValues(final String expected, CSVFormat format) throws IOException {
+ return CSVParser.parse(expected, format).getRecords().get(0).values();
+ }
+
+ @Test
public void testPrinter1() throws IOException {
final StringWriter sw = new StringWriter();
final CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT);
@@ -429,11 +567,28 @@ public class CSVPrinterTest {
}
@Test
- public void testRandom() throws Exception {
- final int iter = 10000;
- doRandom(CSVFormat.DEFAULT, iter);
- doRandom(CSVFormat.EXCEL, iter);
- doRandom(CSVFormat.MYSQL, iter);
+ public void testRandomDefault() throws Exception {
+ doRandom(CSVFormat.DEFAULT, ITERATIONS_FOR_RANDOM_TEST);
+ }
+
+ @Test
+ public void testRandomExcel() throws Exception {
+ doRandom(CSVFormat.EXCEL, ITERATIONS_FOR_RANDOM_TEST);
+ }
+
+ @Test
+ public void testRandomMySql() throws Exception {
+ doRandom(CSVFormat.MYSQL, ITERATIONS_FOR_RANDOM_TEST);
+ }
+
+ @Test
+ public void testRandomTdf() throws Exception {
+ doRandom(CSVFormat.TDF, ITERATIONS_FOR_RANDOM_TEST);
+ }
+
+ @Test
+ public void testRandomRfc4180() throws Exception {
+ doRandom(CSVFormat.RFC4180, ITERATIONS_FOR_RANDOM_TEST);
}
@Test
@@ -497,6 +652,42 @@ public class CSVPrinterTest {
}
@Test
+ public void testEscapeBackslash() throws IOException {
+ StringWriter sw = new StringWriter();
+ final char quoteChar = '\'';
+ final String eol = "\r\n";
+ CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(quoteChar));
+ printer.print("\\");
+ printer.close();
+ assertEquals("'\\'", sw.toString());
+
+ sw = new StringWriter();
+ printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(quoteChar));
+ printer.print("\\\r");
+ printer.close();
+ assertEquals("'\\\r'", sw.toString());
+
+ sw = new StringWriter();
+ printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(quoteChar));
+ printer.print("X\\\r");
+ printer.close();
+ assertEquals("'X\\\r'", sw.toString());
+
+ sw = new StringWriter();
+ printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(quoteChar));
+ printer.printRecord(new Object[] { "\\\r" });
+ printer.close();
+ assertEquals("'\\\r'" + eol, sw.toString());
+
+ sw = new StringWriter();
+ printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(quoteChar));
+ printer.print("\\\\");
+ printer.close();
+ assertEquals("'\\\\'", sw.toString());
+
+ }
+
+ @Test
public void testPlainEscaped() throws IOException {
final StringWriter sw = new StringWriter();
final CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(null).withEscape('!'));
@@ -549,8 +740,7 @@ public class CSVPrinterTest {
@Test
public void testHeader() throws IOException {
final StringWriter sw = new StringWriter();
- final CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(null)
- .withHeader("C1", "C2", "C3"));
+ final CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(null).withHeader("C1", "C2", "C3"));
printer.printRecord("a", "b", "c");
printer.printRecord("x", "y", "z");
assertEquals("C1,C2,C3\r\na,b,c\r\nx,y,z\r\n", sw.toString());
@@ -569,10 +759,10 @@ public class CSVPrinterTest {
@Test
public void testSkipHeaderRecordTrue() throws IOException {
- // functionally identical to testHeaderNotSet, used to test CSV-153
+ // functionally identical to testHeaderNotSet, used to test CSV-153
final StringWriter sw = new StringWriter();
- final CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(null)
- .withHeader("C1", "C2", "C3").withSkipHeaderRecord(true));
+ final CSVPrinter printer = new CSVPrinter(sw,
+ CSVFormat.DEFAULT.withQuote(null).withHeader("C1", "C2", "C3").withSkipHeaderRecord(true));
printer.printRecord("a", "b", "c");
printer.printRecord("x", "y", "z");
assertEquals("a,b,c\r\nx,y,z\r\n", sw.toString());
@@ -581,10 +771,10 @@ public class CSVPrinterTest {
@Test
public void testSkipHeaderRecordFalse() throws IOException {
- // functionally identical to testHeader, used to test CSV-153
+ // functionally identical to testHeader, used to test CSV-153
final StringWriter sw = new StringWriter();
- final CSVPrinter printer = new CSVPrinter(sw, CSVFormat.DEFAULT.withQuote(null)
- .withHeader("C1", "C2", "C3").withSkipHeaderRecord(false));
+ final CSVPrinter printer = new CSVPrinter(sw,
+ CSVFormat.DEFAULT.withQuote(null).withHeader("C1", "C2", "C3").withSkipHeaderRecord(false));
printer.printRecord("a", "b", "c");
printer.printRecord("x", "y", "z");
assertEquals("C1,C2,C3\r\na,b,c\r\nx,y,z\r\n", sw.toString());
@@ -597,7 +787,8 @@ public class CSVPrinterTest {
final Date now = new Date();
final CSVFormat format = CSVFormat.EXCEL;
final CSVPrinter csvPrinter = printWithHeaderComments(sw, now, format);
- assertEquals("# Generated by Apache Commons CSV 1.1\r\n# " + now + "\r\nCol1,Col2\r\nA,B\r\nC,D\r\n", sw.toString());
+ assertEquals("# Generated by Apache Commons CSV 1.1\r\n# " + now + "\r\nCol1,Col2\r\nA,B\r\nC,D\r\n",
+ sw.toString());
csvPrinter.close();
}
@@ -607,7 +798,8 @@ public class CSVPrinterTest {
final Date now = new Date();
final CSVFormat format = CSVFormat.TDF;
final CSVPrinter csvPrinter = printWithHeaderComments(sw, now, format);
- assertEquals("# Generated by Apache Commons CSV 1.1\r\n# " + now + "\r\nCol1\tCol2\r\nA\tB\r\nC\tD\r\n", sw.toString());
+ assertEquals("# Generated by Apache Commons CSV 1.1\r\n# " + now + "\r\nCol1\tCol2\r\nA\tB\r\nC\tD\r\n",
+ sw.toString());
csvPrinter.close();
}