You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2016/06/13 06:56:49 UTC
svn commit: r1748094 [3/3] - in /commons/proper/csv/trunk/src:
main/java/org/apache/commons/csv/ test/java/org/apache/commons/csv/
test/java/org/apache/commons/csv/bugs/ test/java/org/apache/commons/csv/perf/
Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java?rev=1748094&r1=1748093&r2=1748094&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java (original)
+++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java Mon Jun 13 06:56:49 2016
@@ -52,345 +52,341 @@ public class LexerTest {
formatWithEscaping = CSVFormat.DEFAULT.withEscape('\\');
}
- private Lexer getLexer(final String input, final CSVFormat format) {
+ private Lexer createLexer(final String input, final CSVFormat format) {
return new Lexer(format, new ExtendedBufferedReader(new StringReader(input)));
}
@Test
public void testSurroundingSpacesAreDeleted() throws IOException {
final String code = "noSpaces, leadingSpaces,trailingSpaces , surroundingSpaces , ,,";
- final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "noSpaces"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingSpaces"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingSpaces"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingSpaces"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "noSpaces"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingSpaces"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingSpaces"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingSpaces"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ }
}
@Test
public void testSurroundingTabsAreDeleted() throws IOException {
final String code = "noTabs,\tleadingTab,trailingTab\t,\tsurroundingTabs\t,\t\t,,";
- final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "noTabs"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingTab"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingTab"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingTabs"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "noTabs"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingTab"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingTab"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingTabs"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ }
}
@Test
public void testIgnoreEmptyLines() throws IOException {
- final String code =
- "first,line,\n"+
- "\n"+
- "\n"+
- "second,line\n"+
- "\n"+
- "\n"+
- "third line \n"+
- "\n"+
- "\n"+
- "last, line \n"+
- "\n"+
- "\n"+
- "\n";
+ final String code = "first,line,\n" + "\n" + "\n" + "second,line\n" + "\n" + "\n" + "third line \n" + "\n" +
+ "\n" + "last, line \n" + "\n" + "\n" + "\n";
final CSVFormat format = CSVFormat.DEFAULT.withIgnoreEmptyLines();
- final Lexer parser = getLexer(code, format);
-
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "line"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line "));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "last"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, " line "));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ try (final Lexer parser = createLexer(code, format)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "line"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line "));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "last"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, " line "));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ }
}
@Test
public void testComments() throws IOException {
- final String code =
- "first,line,\n"+
- "second,line,tokenWith#no-comment\n"+
- "# comment line \n"+
- "third,line,#no-comment\n"+
- "# penultimate comment\n"+
- "# Final comment\n";
+ final String code = "first,line,\n" + "second,line,tokenWith#no-comment\n" + "# comment line \n" +
+ "third,line,#no-comment\n" + "# penultimate comment\n" + "# Final comment\n";
final CSVFormat format = CSVFormat.DEFAULT.withCommentMarker('#');
- final Lexer parser = getLexer(code, format);
-
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment"));
- assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "third"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment"));
- assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment"));
- assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment"));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ try (final Lexer parser = createLexer(code, format)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment"));
+ assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "third"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment"));
+ assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment"));
+ assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment"));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ }
}
@Test
public void testCommentsAndEmptyLines() throws IOException {
- final String code =
- "1,2,3,\n"+ // 1
- "\n"+ // 1b
- "\n"+ // 1c
- "a,b x,c#no-comment\n"+ // 2
- "#foo\n"+ // 3
- "\n"+ // 4
- "\n"+ // 4b
- "d,e,#no-comment\n"+ // 5
- "\n"+ // 5b
- "\n"+ // 5c
- "# penultimate comment\n"+ // 6
- "\n"+ // 6b
- "\n"+ // 6c
- "# Final comment\n"; // 7
+ final String code = "1,2,3,\n" + // 1
+ "\n" + // 1b
+ "\n" + // 1c
+ "a,b x,c#no-comment\n" + // 2
+ "#foo\n" + // 3
+ "\n" + // 4
+ "\n" + // 4b
+ "d,e,#no-comment\n" + // 5
+ "\n" + // 5b
+ "\n" + // 5c
+ "# penultimate comment\n" + // 6
+ "\n" + // 6b
+ "\n" + // 6c
+ "# Final comment\n"; // 7
final CSVFormat format = CSVFormat.DEFAULT.withCommentMarker('#').withIgnoreEmptyLines(false);
assertFalse("Should not ignore empty lines", format.getIgnoreEmptyLines());
- final Lexer parser = getLexer(code, format);
-
-
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "1"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "2"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "3"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1b
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1c
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2
- assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4b
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "d"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "e"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5b
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5c
- assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6b
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6c
- assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
-
+ try (final Lexer parser = createLexer(code, format)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "1"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "2"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "3"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1b
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1c
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2
+ assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4b
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "d"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "e"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5b
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5c
+ assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6b
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6c
+ assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ }
}
// simple token with escaping not enabled
@Test
public void testBackslashWithoutEscaping() throws IOException {
- /* file: a,\,,b
- * \,,
- */
+ /*
+ * file: a,\,,b \,,
+ */
final String code = "a,\\,,b\\\n\\,,";
final CSVFormat format = CSVFormat.DEFAULT;
assertFalse(format.isEscapeCharacterSet());
- final Lexer parser = getLexer(code, format);
-
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- // an unquoted single backslash is not an escape char
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
- // an unquoted single backslash is not an escape char
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
- assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ try (final Lexer parser = createLexer(code, format)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ // an unquoted single backslash is not an escape char
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
+ // an unquoted single backslash is not an escape char
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
+ assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+ }
}
// simple token with escaping enabled
@Test
public void testBackslashWithEscaping() throws IOException {
- /* file: a,\,,b
- * \,,
- */
+ /*
+ * file: a,\,,b \,,
+ */
final String code = "a,\\,,b\\\\\n\\,,\\\nc,d\\\r\ne";
final CSVFormat format = formatWithEscaping.withIgnoreEmptyLines(false);
assertTrue(format.isEscapeCharacterSet());
- final Lexer parser = getLexer(code, format);
-
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "\nc"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "d\r"));
- assertThat(parser.nextToken(new Token()), matches(EOF, "e"));
+ try (final Lexer parser = createLexer(code, format)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ","));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "\nc"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "d\r"));
+ assertThat(parser.nextToken(new Token()), matches(EOF, "e"));
+ }
}
// encapsulator tokenizer (single line)
@Test
public void testNextToken4() throws IOException {
- /* file: a,"foo",b
- * a, " foo",b
- * a,"foo " ,b // whitespace after closing encapsulator
- * a, " foo " ,b
- */
+ /*
+ * file: a,"foo",b a, " foo",b a,"foo " ,b // whitespace after closing encapsulator a, " foo " ,b
+ */
final String code = "a,\"foo\",b\na, \" foo\",b\na,\"foo \" ,b\na, \" foo \" ,b";
- final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces());
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo "));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo "));
-// assertTokenEquals(EORECORD, "b", parser.nextToken(new Token()));
- assertThat(parser.nextToken(new Token()), matches(EOF, "b"));
+ try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo "));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo "));
+ // assertTokenEquals(EORECORD, "b", parser.nextToken(new Token()));
+ assertThat(parser.nextToken(new Token()), matches(EOF, "b"));
+ }
}
// encapsulator tokenizer (multi line, delimiter in string)
@Test
public void testNextToken5() throws IOException {
final String code = "a,\"foo\n\",b\n\"foo\n baar ,,,\"\n\"\n\t \n\"";
- final Lexer parser = getLexer(code, CSVFormat.DEFAULT);
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo\n"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "foo\n baar ,,,"));
- assertThat(parser.nextToken(new Token()), matches(EOF, "\n\t \n"));
-
+ try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo\n"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "b"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "foo\n baar ,,,"));
+ assertThat(parser.nextToken(new Token()), matches(EOF, "\n\t \n"));
+ }
}
// change delimiters, comment, encapsulater
@Test
public void testNextToken6() throws IOException {
- /* file: a;'b and \' more
- * '
- * !comment;;;;
- * ;;
- */
+ /*
+ * file: a;'b and \' more ' !comment;;;; ;;
+ */
final String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
final CSVFormat format = CSVFormat.DEFAULT.withQuote('\'').withCommentMarker('!').withDelimiter(';');
- final Lexer parser = getLexer(code, format);
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(EORECORD, "b and ' more\n"));
+ try (final Lexer parser = createLexer(code, format)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
+ assertThat(parser.nextToken(new Token()), matches(EORECORD, "b and ' more\n"));
+ }
}
// From CSV-1
@Test
public void testDelimiterIsWhitespace() throws IOException {
final String code = "one\ttwo\t\tfour \t five\t six";
- final Lexer parser = getLexer(code, CSVFormat.TDF);
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "one"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "two"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "four"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "five"));
- assertThat(parser.nextToken(new Token()), matches(EOF, "six"));
+ try (final Lexer parser = createLexer(code, CSVFormat.TDF)) {
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "one"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "two"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, ""));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "four"));
+ assertThat(parser.nextToken(new Token()), matches(TOKEN, "five"));
+ assertThat(parser.nextToken(new Token()), matches(EOF, "six"));
+ }
}
@Test
public void testEscapedCR() throws Exception {
- final Lexer lexer = getLexer("character\\" + CR + "Escaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
+ try (final Lexer lexer = createLexer("character\\" + CR + "Escaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
+ }
}
@Test
public void testCR() throws Exception {
- final Lexer lexer = getLexer("character" + CR + "NotEscaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character"));
- assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
+ try (final Lexer lexer = createLexer("character" + CR + "NotEscaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character"));
+ assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
+ }
}
@Test
public void testEscapedLF() throws Exception {
- final Lexer lexer = getLexer("character\\" + LF + "Escaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + LF + "Escaped"));
+ try (final Lexer lexer = createLexer("character\\" + LF + "Escaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + LF + "Escaped"));
+ }
}
@Test
public void testLF() throws Exception {
- final Lexer lexer = getLexer("character" + LF + "NotEscaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character"));
- assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
+ try (final Lexer lexer = createLexer("character" + LF + "NotEscaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character"));
+ assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped"));
+ }
}
@Test // TODO is this correct? Do we expect <esc>TAB to be unescaped?
public void testEscapedTab() throws Exception {
- final Lexer lexer = getLexer("character\\" + TAB + "Escaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "Escaped"));
+ try (final Lexer lexer = createLexer("character\\" + TAB + "Escaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "Escaped"));
+ }
+
}
@Test
public void testTab() throws Exception {
- final Lexer lexer = getLexer("character" + TAB + "NotEscaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "NotEscaped"));
+ try (final Lexer lexer = createLexer("character" + TAB + "NotEscaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "NotEscaped"));
+ }
}
@Test // TODO is this correct? Do we expect <esc>BACKSPACE to be unescaped?
public void testEscapedBackspace() throws Exception {
- final Lexer lexer = getLexer("character\\" + BACKSPACE + "Escaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "Escaped"));
+ try (final Lexer lexer = createLexer("character\\" + BACKSPACE + "Escaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "Escaped"));
+ }
}
@Test
public void testBackspace() throws Exception {
- final Lexer lexer = getLexer("character" + BACKSPACE + "NotEscaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "NotEscaped"));
+ try (final Lexer lexer = createLexer("character" + BACKSPACE + "NotEscaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "NotEscaped"));
+ }
}
@Test // TODO is this correct? Do we expect <esc>FF to be unescaped?
public void testEscapedFF() throws Exception {
- final Lexer lexer = getLexer("character\\" + FF + "Escaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "Escaped"));
+ try (final Lexer lexer = createLexer("character\\" + FF + "Escaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "Escaped"));
+ }
}
@Test
public void testFF() throws Exception {
- final Lexer lexer = getLexer("character" + FF + "NotEscaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "NotEscaped"));
+ try (final Lexer lexer = createLexer("character" + FF + "NotEscaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "NotEscaped"));
+ }
}
@Test
public void testEscapedMySqlNullValue() throws Exception {
// MySQL uses \N to symbolize null values. We have to restore this
- final Lexer lexer = getLexer("character\\NEscaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character\\NEscaped"));
+ try (final Lexer lexer = createLexer("character\\NEscaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character\\NEscaped"));
+ }
}
@Test
public void testEscapedCharacter() throws Exception {
- final Lexer lexer = getLexer("character\\aEscaped", formatWithEscaping);
- assertThat(lexer.nextToken(new Token()), hasContent("character\\aEscaped"));
+ try (final Lexer lexer = createLexer("character\\aEscaped", formatWithEscaping)) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character\\aEscaped"));
+ }
}
@Test
public void testEscapedControlCharacter() throws Exception {
// we are explicitly using an escape different from \ here
- final Lexer lexer = getLexer("character!rEscaped", CSVFormat.DEFAULT.withEscape('!'));
- assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
+ try (final Lexer lexer = createLexer("character!rEscaped", CSVFormat.DEFAULT.withEscape('!'))) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
+ }
}
@Test
public void testEscapedControlCharacter2() throws Exception {
- final Lexer lexer = getLexer("character\\rEscaped", CSVFormat.DEFAULT.withEscape('\\'));
- assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
+ try (final Lexer lexer = createLexer("character\\rEscaped", CSVFormat.DEFAULT.withEscape('\\'))) {
+ assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped"));
+ }
}
@Test(expected = IOException.class)
public void testEscapingAtEOF() throws Exception {
final String code = "escaping at EOF is evil\\";
- final Lexer lexer = getLexer(code, formatWithEscaping);
-
- lexer.nextToken(new Token());
+ try (final Lexer lexer = createLexer(code, formatWithEscaping)) {
+ lexer.nextToken(new Token());
+ }
}
}
Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java?rev=1748094&r1=1748093&r2=1748094&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java (original)
+++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java Mon Jun 13 06:56:49 2016
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.util.zip.GZIPInputStream;
import org.apache.commons.io.IOUtils;
@@ -73,11 +74,11 @@ public class PerformanceTest {
System.out.println(String.format("Found test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
} else {
System.out.println("Decompressing test fixture " + BIG_FILE + "...");
- final InputStream input = new GZIPInputStream(new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
- final OutputStream output = new FileOutputStream(BIG_FILE);
- IOUtils.copy(input, output);
- input.close();
- output.close();
+ try (final InputStream input = new GZIPInputStream(
+ new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
+ final OutputStream output = new FileOutputStream(BIG_FILE)) {
+ IOUtils.copy(input, output);
+ }
System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
}
final int argc = args.length;
@@ -121,7 +122,7 @@ public class PerformanceTest {
}
}
- private static BufferedReader getReader() throws IOException {
+ private static BufferedReader createReader() throws IOException {
return new BufferedReader(new FileReader(BIG_FILE));
}
@@ -155,15 +156,17 @@ public class PerformanceTest {
}
private static void testReadBigFile(final boolean split) throws Exception {
- for (int i = 0; i < max; i++) {
- final BufferedReader in = getReader();
- final long t0 = System.currentTimeMillis();
- final Stats s = readAll(in, split);
- in.close();
- show(split?"file+split":"file", s, t0);
- }
- show();
- }
+ for (int i = 0; i < max; i++) {
+ final long startMillis;
+ final Stats stats;
+ try (final BufferedReader in = createReader()) {
+ startMillis = System.currentTimeMillis();
+ stats = readAll(in, split);
+ }
+ show(split ? "file+split" : "file", stats, startMillis);
+ }
+ show();
+ }
private static Stats readAll(final BufferedReader in, final boolean split) throws IOException {
int count = 0;
@@ -176,55 +179,58 @@ public class PerformanceTest {
return new Stats(count, fields);
}
- private static void testExtendedBuffer(final boolean makeString) throws Exception {
- for (int i = 0; i < max; i++) {
- final ExtendedBufferedReader in = new ExtendedBufferedReader(getReader());
- final long t0 = System.currentTimeMillis();
- int read;
- int fields = 0;
- int lines = 0;
- if (makeString) {
- StringBuilder sb = new StringBuilder();
- while((read=in.read()) != -1) {
- sb.append((char)read);
- if (read == ',') { // count delimiters
- sb.toString();
- sb = new StringBuilder();
- fields++;
- } else if (read == '\n') {
- sb.toString();
- sb = new StringBuilder();
- lines++;
- }
- }
- } else {
- while((read=in.read()) != -1) {
- if (read == ',') { // count delimiters
- fields++;
- } else if (read == '\n') {
- lines++;
- }
- }
- }
- fields += lines; // EOL is a delimiter too
- in.close();
- show("Extended"+(makeString?" toString":""), new Stats(lines, fields), t0);
- }
- show();
- }
+ private static void testExtendedBuffer(final boolean makeString) throws Exception {
+ for (int i = 0; i < max; i++) {
+ int fields = 0;
+ int lines = 0;
+ final long startMillis;
+ try (final ExtendedBufferedReader in = new ExtendedBufferedReader(createReader())) {
+ startMillis = System.currentTimeMillis();
+ int read;
+ if (makeString) {
+ StringBuilder sb = new StringBuilder();
+ while ((read = in.read()) != -1) {
+ sb.append((char) read);
+ if (read == ',') { // count delimiters
+ sb.toString();
+ sb = new StringBuilder();
+ fields++;
+ } else if (read == '\n') {
+ sb.toString();
+ sb = new StringBuilder();
+ lines++;
+ }
+ }
+ } else {
+ while ((read = in.read()) != -1) {
+ if (read == ',') { // count delimiters
+ fields++;
+ } else if (read == '\n') {
+ lines++;
+ }
+ }
+ }
+ fields += lines; // EOL is a delimiter too
+ }
+ show("Extended" + (makeString ? " toString" : ""), new Stats(lines, fields), startMillis);
+ }
+ show();
+ }
- private static void testParseCommonsCSV() throws Exception {
- for (int i = 0; i < max; i++) {
- final BufferedReader reader = getReader();
- final CSVParser parser = new CSVParser(reader, format);
- final long t0 = System.currentTimeMillis();
- final Stats s = iterate(parser);
- reader.close();
- show("CSV", s, t0);
- parser.close();
- }
- show();
- }
+ private static void testParseCommonsCSV() throws Exception {
+ for (int i = 0; i < max; i++) {
+ final long startMillis;
+ final Stats stats;
+ try (final BufferedReader reader = createReader()) {
+ try (final CSVParser parser = new CSVParser(reader, format)) {
+ startMillis = System.currentTimeMillis();
+ stats = iterate(parser);
+ }
+ show("CSV", stats, startMillis);
+ }
+ }
+ show();
+ }
private static Constructor<Lexer> getLexerCtor(final String clazz) throws Exception {
@@ -233,53 +239,59 @@ public class PerformanceTest {
return lexer.getConstructor(new Class<?>[]{CSVFormat.class, ExtendedBufferedReader.class});
}
- private static void testCSVLexer(final boolean newToken, final String test) throws Exception {
- Token token = new Token();
- String dynamic = "";
- for (int i = 0; i < max; i++) {
- final ExtendedBufferedReader input = new ExtendedBufferedReader(getReader());
- Lexer lexer = null;
- if (test.startsWith("CSVLexer")) {
- dynamic="!";
- lexer = getLexerCtor(test).newInstance(new Object[]{format, input});
- } else {
- lexer = new Lexer(format, input);
- }
- int count = 0;
- int fields = 0;
- final long t0 = System.currentTimeMillis();
- do {
- if (newToken) {
- token = new Token();
- } else {
- token.reset();
- }
- lexer.nextToken(token);
- switch(token.type) {
- case EOF:
- break;
- case EORECORD:
- fields++;
- count++;
- break;
- case INVALID:
- throw new IOException("invalid parse sequence <"+token.content.toString()+">");
- case TOKEN:
- fields++;
- break;
- case COMMENT: // not really expecting these
- break;
- default:
- throw new IllegalStateException("Unexpected Token type: " + token.type);
- }
-
- } while (!token.type.equals(Token.Type.EOF));
- final Stats s = new Stats(count, fields);
- input.close();
- show(lexer.getClass().getSimpleName()+dynamic+" "+(newToken ? "new" : "reset"), s, t0);
- }
- show();
- }
+ private static void testCSVLexer(final boolean newToken, final String test) throws Exception {
+ Token token = new Token();
+ String dynamic = "";
+ for (int i = 0; i < max; i++) {
+ final String simpleName;
+ final Stats stats;
+ final long startMillis;
+ try (final ExtendedBufferedReader input = new ExtendedBufferedReader(createReader());
+ Lexer lexer = createTestCSVLexer(test, input)) {
+ if (test.startsWith("CSVLexer")) {
+ dynamic = "!";
+ }
+ simpleName = lexer.getClass().getSimpleName();
+ int count = 0;
+ int fields = 0;
+ startMillis = System.currentTimeMillis();
+ do {
+ if (newToken) {
+ token = new Token();
+ } else {
+ token.reset();
+ }
+ lexer.nextToken(token);
+ switch (token.type) {
+ case EOF:
+ break;
+ case EORECORD:
+ fields++;
+ count++;
+ break;
+ case INVALID:
+ throw new IOException("invalid parse sequence <" + token.content.toString() + ">");
+ case TOKEN:
+ fields++;
+ break;
+ case COMMENT: // not really expecting these
+ break;
+ default:
+ throw new IllegalStateException("Unexpected Token type: " + token.type);
+ }
+ } while (!token.type.equals(Token.Type.EOF));
+ stats = new Stats(count, fields);
+ }
+ show(simpleName + dynamic + " " + (newToken ? "new" : "reset"), stats, startMillis);
+ }
+ show();
+ }
+
+ private static Lexer createTestCSVLexer(final String test, final ExtendedBufferedReader input)
+ throws InstantiationException, IllegalAccessException, InvocationTargetException, Exception {
+ return test.startsWith("CSVLexer") ? getLexerCtor(test)
+ .newInstance(new Object[] { format, input }) : new Lexer(format, input);
+ }
private static Stats iterate(final Iterable<CSVRecord> it) {
int count = 0;
Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java?rev=1748094&r1=1748093&r2=1748094&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java (original)
+++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java Mon Jun 13 06:56:49 2016
@@ -29,12 +29,13 @@ public class JiraCsv164Test {
@Test
public void testJiraCsv154_withCommentMarker() throws IOException {
final String comment = "This is a header comment";
- final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withCommentMarker('#').withHeaderComments(comment);
+ final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withCommentMarker('#')
+ .withHeaderComments(comment);
final StringBuilder out = new StringBuilder();
- final CSVPrinter printer = format.print(out);
- printer.print("A");
- printer.print("B");
- printer.close();
+ try (final CSVPrinter printer = format.print(out)) {
+ printer.print("A");
+ printer.print("B");
+ }
final String s = out.toString();
assertTrue(s, s.contains(comment));
}
@@ -42,12 +43,13 @@ public class JiraCsv164Test {
@Test
public void testJiraCsv154_withHeaderComments() throws IOException {
final String comment = "This is a header comment";
- final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withHeaderComments(comment).withCommentMarker('#');
+ final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withHeaderComments(comment)
+ .withCommentMarker('#');
final StringBuilder out = new StringBuilder();
- final CSVPrinter printer = format.print(out);
- printer.print("A");
- printer.print("B");
- printer.close();
+ try (final CSVPrinter printer = format.print(out)) {
+ printer.print("A");
+ printer.print("B");
+ }
final String s = out.toString();
assertTrue(s, s.contains(comment));
}
Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java?rev=1748094&r1=1748093&r2=1748094&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java (original)
+++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java Mon Jun 13 06:56:49 2016
@@ -33,23 +33,23 @@ public class JiraCsv167Test {
@Test
public void parse() throws IOException {
- final BufferedReader br = new BufferedReader(getTestInput());
- String s = null;
int totcomment = 0;
int totrecs = 0;
- boolean lastWasComment = false;
- while((s=br.readLine()) != null) {
- if (s.startsWith("#")) {
- if (!lastWasComment) { // comments are merged
- totcomment++;
+ try (final BufferedReader br = new BufferedReader(getTestInput())) {
+ String s = null;
+ boolean lastWasComment = false;
+ while ((s = br.readLine()) != null) {
+ if (s.startsWith("#")) {
+ if (!lastWasComment) { // comments are merged
+ totcomment++;
+ }
+ lastWasComment = true;
+ } else {
+ totrecs++;
+ lastWasComment = false;
}
- lastWasComment = true;
- } else {
- totrecs++;
- lastWasComment = false;
}
}
- br.close();
CSVFormat format = CSVFormat.DEFAULT;
//
format = format.withAllowMissingColumnNames(false);
@@ -66,13 +66,14 @@ public class JiraCsv167Test {
format = format.withRecordSeparator('\n');
format = format.withSkipHeaderRecord(false);
//
- final CSVParser parser = format.parse(getTestInput());
int comments = 0;
int records = 0;
- for (final CSVRecord csvRecord : parser) {
- records++;
- if (csvRecord.hasComment()) {
- comments++;
+ try (final CSVParser parser = format.parse(getTestInput())) {
+ for (final CSVRecord csvRecord : parser) {
+ records++;
+ if (csvRecord.hasComment()) {
+ comments++;
+ }
}
}
// Comment lines are concatenated, in this example 4 lines become 2 comments.
Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java
URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java?rev=1748094&r1=1748093&r2=1748094&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java (original)
+++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java Mon Jun 13 06:56:49 2016
@@ -56,15 +56,15 @@ public class PerformanceTest {
return;
}
System.out.println("Decompressing test fixture " + BIG_FILE + "...");
- final InputStream input = new GZIPInputStream(new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
- final OutputStream output = new FileOutputStream(BIG_FILE);
- IOUtils.copy(input, output);
- System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
- input.close();
- output.close();
+ try (final InputStream input = new GZIPInputStream(
+ new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz"));
+ final OutputStream output = new FileOutputStream(BIG_FILE)) {
+ IOUtils.copy(input, output);
+ System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
+ }
}
- private BufferedReader getBufferedReader() throws IOException {
+ private BufferedReader createBufferedReader() throws IOException {
return new BufferedReader(new FileReader(BIG_FILE));
}
@@ -96,7 +96,7 @@ public class PerformanceTest {
public long testParseBigFile(final boolean traverseColumns) throws Exception {
final long startMillis = System.currentTimeMillis();
- final long count = this.parse(this.getBufferedReader(), traverseColumns);
+ final long count = this.parse(this.createBufferedReader(), traverseColumns);
final long totalMillis = System.currentTimeMillis() - startMillis;
this.println(String.format("File parsed in %,d milliseconds with Commons CSV: %,d lines.", totalMillis, count));
return totalMillis;
@@ -115,13 +115,12 @@ public class PerformanceTest {
public void testReadBigFile() throws Exception {
long bestTime = Long.MAX_VALUE;
for (int i = 0; i < this.max; i++) {
- final BufferedReader in = this.getBufferedReader();
- final long startMillis = System.currentTimeMillis();
- long count = 0;
- try {
+ final long startMillis;
+ long count;
+ try (final BufferedReader in = this.createBufferedReader()) {
+ startMillis = System.currentTimeMillis();
+ count = 0;
count = this.readAll(in);
- } finally {
- in.close();
}
final long totalMillis = System.currentTimeMillis() - startMillis;
bestTime = Math.min(totalMillis, bestTime);