You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2023/01/21 19:35:32 UTC
[commons-csv] branch master updated (b1bdb99c -> 1269c133)
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
from b1bdb99c Merge pull request #295 from DamjanJovanovic/master
new d81528fb Revert "Add a setting that controls whether the last field on the last line, if quoted, has to have a closing quote before the file ends."
new c22ff413 Revert "Add support for trailing text after the closing quote, for Excel compatibility."
new 1269c133 Tests CSV-141 and PR 295
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/commons/csv/CSVFormat.java | 80 ++-------------
src/main/java/org/apache/commons/csv/Lexer.java | 28 ++----
.../java/org/apache/commons/csv/CSVParserTest.java | 111 +++++++++++++++++++++
.../java/org/apache/commons/csv/LexerTest.java | 26 -----
.../org/apache/commons/csv/CSV-141/csv-141.csv | 4 +
5 files changed, 129 insertions(+), 120 deletions(-)
create mode 100644 src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv
[commons-csv] 02/03: Revert "Add support for trailing text after the closing quote, for Excel compatibility."
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
commit c22ff413ac477a0e43c723ec4e265e211f028de0
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Sat Jan 21 14:34:46 2023 -0500
Revert "Add support for trailing text after the closing quote, for Excel compatibility."
This reverts commit ed0ca2232105f6df1f3fc68762f03c7595dc20c2.
---
.../java/org/apache/commons/csv/CSVFormat.java | 48 ++++------------------
src/main/java/org/apache/commons/csv/Lexer.java | 14 ++-----
.../java/org/apache/commons/csv/LexerTest.java | 13 ------
3 files changed, 11 insertions(+), 64 deletions(-)
diff --git a/src/main/java/org/apache/commons/csv/CSVFormat.java b/src/main/java/org/apache/commons/csv/CSVFormat.java
index 1cdd73c4..81b6f193 100644
--- a/src/main/java/org/apache/commons/csv/CSVFormat.java
+++ b/src/main/java/org/apache/commons/csv/CSVFormat.java
@@ -208,8 +208,6 @@ public final class CSVFormat implements Serializable {
private boolean allowMissingColumnNames;
- private boolean allowTrailingText;
-
private boolean autoFlush;
private Character commentMarker;
@@ -266,7 +264,6 @@ public final class CSVFormat implements Serializable {
this.autoFlush = csvFormat.autoFlush;
this.quotedNullString = csvFormat.quotedNullString;
this.duplicateHeaderMode = csvFormat.duplicateHeaderMode;
- this.allowTrailingText = csvFormat.allowTrailingText;
}
/**
@@ -304,20 +301,6 @@ public final class CSVFormat implements Serializable {
return this;
}
- /**
- * Sets whether to allow trailing text in a quoted field, after the closing quote.
- *
- * @param allowTrailingText the trailing text behavior, {@code true} to append that text to the field contents, {@code false} to throw
- * an {@link IOException}.
- *
- * @return This instance.
- * @since 1.10.0
- */
- public Builder setAllowTrailingText(final boolean allowTrailingText) {
- this.allowTrailingText = allowTrailingText;
- return this;
- }
-
/**
* Sets whether to flush on close.
*
@@ -827,7 +810,7 @@ public final class CSVFormat implements Serializable {
* @see Predefined#Default
*/
public static final CSVFormat DEFAULT = new CSVFormat(COMMA, DOUBLE_QUOTE_CHAR, null, null, null, false, true, CRLF, null, null, null, false, false, false,
- false, false, false, DuplicateHeaderMode.ALLOW_ALL, false);
+ false, false, false, DuplicateHeaderMode.ALLOW_ALL);
/**
* Excel file format (using a comma as the value delimiter). Note that the actual value delimiter used by Excel is locale dependent, it might be necessary
@@ -851,7 +834,6 @@ public final class CSVFormat implements Serializable {
* <li>{@code setIgnoreEmptyLines(false)}</li>
* <li>{@code setAllowMissingColumnNames(true)}</li>
* <li>{@code setDuplicateHeaderMode(DuplicateHeaderMode.ALLOW_ALL)}</li>
- * <li>{@code setAllowTrailingText(true)}</li>
* </ul>
* <p>
* Note: This is currently like {@link #RFC4180} plus {@link Builder#setAllowMissingColumnNames(boolean) Builder#setAllowMissingColumnNames(true)} and
@@ -864,7 +846,6 @@ public final class CSVFormat implements Serializable {
public static final CSVFormat EXCEL = DEFAULT.builder()
.setIgnoreEmptyLines(false)
.setAllowMissingColumnNames(true)
- .setAllowTrailingText(true)
.build();
// @formatter:on
@@ -1287,7 +1268,7 @@ public final class CSVFormat implements Serializable {
*/
public static CSVFormat newFormat(final char delimiter) {
return new CSVFormat(String.valueOf(delimiter), null, null, null, null, false, false, null, null, null, null, false, false, false, false, false, false,
- DuplicateHeaderMode.ALLOW_ALL, false);
+ DuplicateHeaderMode.ALLOW_ALL);
}
static String[] toStringArray(final Object[] values) {
@@ -1331,8 +1312,6 @@ public final class CSVFormat implements Serializable {
private final boolean allowMissingColumnNames;
- private final boolean allowTrailingText;
-
private final boolean autoFlush;
private final Character commentMarker; // null if commenting is disabled
@@ -1387,7 +1366,6 @@ public final class CSVFormat implements Serializable {
this.autoFlush = builder.autoFlush;
this.quotedNullString = builder.quotedNullString;
this.duplicateHeaderMode = builder.duplicateHeaderMode;
- this.allowTrailingText = builder.allowTrailingText;
validate();
}
@@ -1418,7 +1396,7 @@ public final class CSVFormat implements Serializable {
final boolean ignoreSurroundingSpaces, final boolean ignoreEmptyLines, final String recordSeparator, final String nullString,
final Object[] headerComments, final String[] header, final boolean skipHeaderRecord, final boolean allowMissingColumnNames,
final boolean ignoreHeaderCase, final boolean trim, final boolean trailingDelimiter, final boolean autoFlush,
- final DuplicateHeaderMode duplicateHeaderMode, final boolean allowTrailingText) {
+ final DuplicateHeaderMode duplicateHeaderMode) {
this.delimiter = delimiter;
this.quoteCharacter = quoteChar;
this.quoteMode = quoteMode;
@@ -1438,7 +1416,6 @@ public final class CSVFormat implements Serializable {
this.autoFlush = autoFlush;
this.quotedNullString = quoteCharacter + nullString + quoteCharacter;
this.duplicateHeaderMode = duplicateHeaderMode;
- this.allowTrailingText = allowTrailingText;
validate();
}
@@ -1492,8 +1469,7 @@ public final class CSVFormat implements Serializable {
ignoreHeaderCase == other.ignoreHeaderCase && ignoreSurroundingSpaces == other.ignoreSurroundingSpaces &&
Objects.equals(nullString, other.nullString) && Objects.equals(quoteCharacter, other.quoteCharacter) && quoteMode == other.quoteMode &&
Objects.equals(quotedNullString, other.quotedNullString) && Objects.equals(recordSeparator, other.recordSeparator) &&
- skipHeaderRecord == other.skipHeaderRecord && trailingDelimiter == other.trailingDelimiter && trim == other.trim &&
- allowTrailingText == other.allowTrailingText;
+ skipHeaderRecord == other.skipHeaderRecord && trailingDelimiter == other.trailingDelimiter && trim == other.trim;
}
/**
@@ -1536,16 +1512,6 @@ public final class CSVFormat implements Serializable {
return allowMissingColumnNames;
}
- /**
- * Gets whether quoted fields allow trailing text after the closing quote.
- *
- * @return {@code true} if allowed, {@code false} to throw an {@link IOException}.
- * @since 1.10.0
- */
- public boolean getAllowTrailingText() {
- return allowTrailingText;
- }
-
/**
* Gets whether to flush on close.
*
@@ -1726,9 +1692,9 @@ public final class CSVFormat implements Serializable {
int result = 1;
result = prime * result + Arrays.hashCode(headers);
result = prime * result + Arrays.hashCode(headerComments);
- return prime * result + Objects.hash(duplicateHeaderMode, allowMissingColumnNames, allowTrailingText, autoFlush, commentMarker, delimiter,
- escapeCharacter, ignoreEmptyLines, ignoreHeaderCase, ignoreSurroundingSpaces, nullString, quoteCharacter, quoteMode, quotedNullString,
- recordSeparator, skipHeaderRecord, trailingDelimiter, trim);
+ return prime * result + Objects.hash(duplicateHeaderMode, allowMissingColumnNames, autoFlush, commentMarker, delimiter, escapeCharacter,
+ ignoreEmptyLines, ignoreHeaderCase, ignoreSurroundingSpaces, nullString, quoteCharacter, quoteMode, quotedNullString, recordSeparator,
+ skipHeaderRecord, trailingDelimiter, trim);
}
/**
diff --git a/src/main/java/org/apache/commons/csv/Lexer.java b/src/main/java/org/apache/commons/csv/Lexer.java
index fd60b5ac..06b2c9c2 100644
--- a/src/main/java/org/apache/commons/csv/Lexer.java
+++ b/src/main/java/org/apache/commons/csv/Lexer.java
@@ -57,7 +57,6 @@ final class Lexer implements Closeable {
private final boolean ignoreSurroundingSpaces;
private final boolean ignoreEmptyLines;
- private final boolean allowTrailingText;
/** The input stream */
private final ExtendedBufferedReader reader;
@@ -73,7 +72,6 @@ final class Lexer implements Closeable {
this.commentStart = mapNullToDisabled(format.getCommentMarker());
this.ignoreSurroundingSpaces = format.getIgnoreSurroundingSpaces();
this.ignoreEmptyLines = format.getIgnoreEmptyLines();
- this.allowTrailingText = format.getAllowTrailingText();
this.delimiterBuf = new char[delimiter.length - 1];
this.escapeDelimiterBuf = new char[2 * delimiter.length - 1];
}
@@ -366,14 +364,10 @@ final class Lexer implements Closeable {
token.type = EORECORD;
return token;
}
- if (allowTrailingText) {
- token.content.append((char) c);
- } else {
- if (!Character.isWhitespace((char)c)) {
- // error invalid char between token and next delimiter
- throw new IOException("(line " + getCurrentLineNumber() +
- ") invalid char between encapsulated token and delimiter");
- }
+ if (!Character.isWhitespace((char)c)) {
+ // error invalid char between token and next delimiter
+ throw new IOException("(line " + getCurrentLineNumber() +
+ ") invalid char between encapsulated token and delimiter");
}
}
}
diff --git a/src/test/java/org/apache/commons/csv/LexerTest.java b/src/test/java/org/apache/commons/csv/LexerTest.java
index 7edc7d86..cc8d728a 100644
--- a/src/test/java/org/apache/commons/csv/LexerTest.java
+++ b/src/test/java/org/apache/commons/csv/LexerTest.java
@@ -431,17 +431,4 @@ public class LexerTest {
lexer.trimTrailingSpaces(buffer);
assertThat(lexer.nextToken(new Token()), matches(EOF, ""));
}
-
- @Test
- public void testTrailingTextAfterQuote() throws Exception {
- final String code = "\"a\" b,\"a\" \" b,\"a\" b \"\"";
- try (final Lexer parser = createLexer(code, CSVFormat.Builder.create().setAllowTrailingText(true).build())) {
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a b"));
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a \" b"));
- assertThat(parser.nextToken(new Token()), matches(EOF, "a b \"\""));
- }
- try (final Lexer parser = createLexer(code, CSVFormat.Builder.create().setAllowTrailingText(false).build())) {
- assertThrows(IOException.class, () -> lexer.nextToken(new Token()));
- }
- }
}
[commons-csv] 01/03: Revert "Add a setting that controls whether the last field on the last line, if quoted, has to have a closing quote before the file ends."
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
commit d81528fb814cfa2675966e163b6885d6b7cb9e58
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Sat Jan 21 14:34:46 2023 -0500
Revert "Add a setting that controls whether the last field on the last line, if quoted, has to have a closing quote before the file ends."
This reverts commit d0ea9e3a000aa358a4960df6cfc8abd735a3d165.
---
.../java/org/apache/commons/csv/CSVFormat.java | 46 ++++------------------
src/main/java/org/apache/commons/csv/Lexer.java | 14 ++-----
.../java/org/apache/commons/csv/LexerTest.java | 15 +------
3 files changed, 11 insertions(+), 64 deletions(-)
diff --git a/src/main/java/org/apache/commons/csv/CSVFormat.java b/src/main/java/org/apache/commons/csv/CSVFormat.java
index 280f99f2..1cdd73c4 100644
--- a/src/main/java/org/apache/commons/csv/CSVFormat.java
+++ b/src/main/java/org/apache/commons/csv/CSVFormat.java
@@ -206,8 +206,6 @@ public final class CSVFormat implements Serializable {
return new Builder(csvFormat);
}
- private boolean allowEofWithoutClosingQuote;
-
private boolean allowMissingColumnNames;
private boolean allowTrailingText;
@@ -269,7 +267,6 @@ public final class CSVFormat implements Serializable {
this.quotedNullString = csvFormat.quotedNullString;
this.duplicateHeaderMode = csvFormat.duplicateHeaderMode;
this.allowTrailingText = csvFormat.allowTrailingText;
- this.allowEofWithoutClosingQuote = csvFormat.allowEofWithoutClosingQuote;
}
/**
@@ -294,19 +291,6 @@ public final class CSVFormat implements Serializable {
return this;
}
- /**
- * Sets whether the last field on the last line, if quoted, can have no closing quote when the file ends, {@code true} if this is ok,
- * {@code false} if {@link IOException} should be thrown.
- *
- * @param allowEofWithoutClosingQuote whether to allow the last field on the last line to have a missing closing quote when the file ends,
- * {@code true} if so, or {@code false} to cause an {@link IOException} to be thrown.
- * @since 1.10.0
- */
- public Builder setAllowEofWithoutClosingQuote(final boolean allowEofWithoutClosingQuote) {
- this.allowEofWithoutClosingQuote = allowEofWithoutClosingQuote;
- return this;
- }
-
/**
* Sets the parser missing column names behavior, {@code true} to allow missing column names in the header line, {@code false} to cause an
* {@link IllegalArgumentException} to be thrown.
@@ -843,7 +827,7 @@ public final class CSVFormat implements Serializable {
* @see Predefined#Default
*/
public static final CSVFormat DEFAULT = new CSVFormat(COMMA, DOUBLE_QUOTE_CHAR, null, null, null, false, true, CRLF, null, null, null, false, false, false,
- false, false, false, DuplicateHeaderMode.ALLOW_ALL, false, false);
+ false, false, false, DuplicateHeaderMode.ALLOW_ALL, false);
/**
* Excel file format (using a comma as the value delimiter). Note that the actual value delimiter used by Excel is locale dependent, it might be necessary
@@ -868,7 +852,6 @@ public final class CSVFormat implements Serializable {
* <li>{@code setAllowMissingColumnNames(true)}</li>
* <li>{@code setDuplicateHeaderMode(DuplicateHeaderMode.ALLOW_ALL)}</li>
* <li>{@code setAllowTrailingText(true)}</li>
- * <li>{@code setAllowEofWithoutClosingQuote(true)}</li>
* </ul>
* <p>
* Note: This is currently like {@link #RFC4180} plus {@link Builder#setAllowMissingColumnNames(boolean) Builder#setAllowMissingColumnNames(true)} and
@@ -882,7 +865,6 @@ public final class CSVFormat implements Serializable {
.setIgnoreEmptyLines(false)
.setAllowMissingColumnNames(true)
.setAllowTrailingText(true)
- .setAllowEofWithoutClosingQuote(true)
.build();
// @formatter:on
@@ -1305,7 +1287,7 @@ public final class CSVFormat implements Serializable {
*/
public static CSVFormat newFormat(final char delimiter) {
return new CSVFormat(String.valueOf(delimiter), null, null, null, null, false, false, null, null, null, null, false, false, false, false, false, false,
- DuplicateHeaderMode.ALLOW_ALL, false, false);
+ DuplicateHeaderMode.ALLOW_ALL, false);
}
static String[] toStringArray(final Object[] values) {
@@ -1347,8 +1329,6 @@ public final class CSVFormat implements Serializable {
private final DuplicateHeaderMode duplicateHeaderMode;
- private final boolean allowEofWithoutClosingQuote;
-
private final boolean allowMissingColumnNames;
private final boolean allowTrailingText;
@@ -1408,7 +1388,6 @@ public final class CSVFormat implements Serializable {
this.quotedNullString = builder.quotedNullString;
this.duplicateHeaderMode = builder.duplicateHeaderMode;
this.allowTrailingText = builder.allowTrailingText;
- this.allowEofWithoutClosingQuote = builder.allowEofWithoutClosingQuote;
validate();
}
@@ -1439,7 +1418,7 @@ public final class CSVFormat implements Serializable {
final boolean ignoreSurroundingSpaces, final boolean ignoreEmptyLines, final String recordSeparator, final String nullString,
final Object[] headerComments, final String[] header, final boolean skipHeaderRecord, final boolean allowMissingColumnNames,
final boolean ignoreHeaderCase, final boolean trim, final boolean trailingDelimiter, final boolean autoFlush,
- final DuplicateHeaderMode duplicateHeaderMode, final boolean allowTrailingText, final boolean allowEofWithoutClosingQuote) {
+ final DuplicateHeaderMode duplicateHeaderMode, final boolean allowTrailingText) {
this.delimiter = delimiter;
this.quoteCharacter = quoteChar;
this.quoteMode = quoteMode;
@@ -1460,7 +1439,6 @@ public final class CSVFormat implements Serializable {
this.quotedNullString = quoteCharacter + nullString + quoteCharacter;
this.duplicateHeaderMode = duplicateHeaderMode;
this.allowTrailingText = allowTrailingText;
- this.allowEofWithoutClosingQuote = allowEofWithoutClosingQuote;
validate();
}
@@ -1515,7 +1493,7 @@ public final class CSVFormat implements Serializable {
Objects.equals(nullString, other.nullString) && Objects.equals(quoteCharacter, other.quoteCharacter) && quoteMode == other.quoteMode &&
Objects.equals(quotedNullString, other.quotedNullString) && Objects.equals(recordSeparator, other.recordSeparator) &&
skipHeaderRecord == other.skipHeaderRecord && trailingDelimiter == other.trailingDelimiter && trim == other.trim &&
- allowTrailingText == other.allowTrailingText && allowEofWithoutClosingQuote == other.allowEofWithoutClosingQuote;
+ allowTrailingText == other.allowTrailingText;
}
/**
@@ -1549,16 +1527,6 @@ public final class CSVFormat implements Serializable {
return duplicateHeaderMode == DuplicateHeaderMode.ALLOW_ALL;
}
- /**
- * Gets whether the file can end before the last field on the last line, if quoted, has a closing quote.
- *
- * @return {@code true} if so, {@code false} to throw an {@link IOException}.
- * @since 1.10.0
- */
- public boolean getAllowEofWithoutClosingQuote() {
- return allowEofWithoutClosingQuote;
- }
-
/**
* Gets whether missing column names are allowed when parsing the header line.
*
@@ -1758,9 +1726,9 @@ public final class CSVFormat implements Serializable {
int result = 1;
result = prime * result + Arrays.hashCode(headers);
result = prime * result + Arrays.hashCode(headerComments);
- return prime * result + Objects.hash(duplicateHeaderMode, allowEofWithoutClosingQuote, allowMissingColumnNames, allowTrailingText,
- autoFlush, commentMarker, delimiter, escapeCharacter, ignoreEmptyLines, ignoreHeaderCase, ignoreSurroundingSpaces,
- nullString, quoteCharacter, quoteMode, quotedNullString, recordSeparator, skipHeaderRecord, trailingDelimiter, trim);
+ return prime * result + Objects.hash(duplicateHeaderMode, allowMissingColumnNames, allowTrailingText, autoFlush, commentMarker, delimiter,
+ escapeCharacter, ignoreEmptyLines, ignoreHeaderCase, ignoreSurroundingSpaces, nullString, quoteCharacter, quoteMode, quotedNullString,
+ recordSeparator, skipHeaderRecord, trailingDelimiter, trim);
}
/**
diff --git a/src/main/java/org/apache/commons/csv/Lexer.java b/src/main/java/org/apache/commons/csv/Lexer.java
index c43c52ed..fd60b5ac 100644
--- a/src/main/java/org/apache/commons/csv/Lexer.java
+++ b/src/main/java/org/apache/commons/csv/Lexer.java
@@ -58,7 +58,6 @@ final class Lexer implements Closeable {
private final boolean ignoreSurroundingSpaces;
private final boolean ignoreEmptyLines;
private final boolean allowTrailingText;
- private final boolean allowEofWithoutClosingQuote;
/** The input stream */
private final ExtendedBufferedReader reader;
@@ -75,7 +74,6 @@ final class Lexer implements Closeable {
this.ignoreSurroundingSpaces = format.getIgnoreSurroundingSpaces();
this.ignoreEmptyLines = format.getIgnoreEmptyLines();
this.allowTrailingText = format.getAllowTrailingText();
- this.allowEofWithoutClosingQuote = format.getAllowEofWithoutClosingQuote();
this.delimiterBuf = new char[delimiter.length - 1];
this.escapeDelimiterBuf = new char[2 * delimiter.length - 1];
}
@@ -380,15 +378,9 @@ final class Lexer implements Closeable {
}
}
} else if (isEndOfFile(c)) {
- if (allowEofWithoutClosingQuote) {
- token.type = EOF;
- token.isReady = true; // There is data at EOF
- return token;
- } else {
- // error condition (end of file before end of token)
- throw new IOException("(startline " + startLineNumber +
- ") EOF reached before encapsulated token finished");
- }
+ // error condition (end of file before end of token)
+ throw new IOException("(startline " + startLineNumber +
+ ") EOF reached before encapsulated token finished");
} else {
// consume character
token.content.append((char) c);
diff --git a/src/test/java/org/apache/commons/csv/LexerTest.java b/src/test/java/org/apache/commons/csv/LexerTest.java
index 85199072..7edc7d86 100644
--- a/src/test/java/org/apache/commons/csv/LexerTest.java
+++ b/src/test/java/org/apache/commons/csv/LexerTest.java
@@ -441,20 +441,7 @@ public class LexerTest {
assertThat(parser.nextToken(new Token()), matches(EOF, "a b \"\""));
}
try (final Lexer parser = createLexer(code, CSVFormat.Builder.create().setAllowTrailingText(false).build())) {
- assertThrows(IOException.class, () -> parser.nextToken(new Token()));
- }
- }
-
- @Test
- public void testEOFWithoutClosingQuote() throws Exception {
- final String code = "a,\"b";
- try (final Lexer parser = createLexer(code, CSVFormat.Builder.create().setAllowEofWithoutClosingQuote(true).build())) {
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThat(parser.nextToken(new Token()), matches(EOF, "b"));
- }
- try (final Lexer parser = createLexer(code, CSVFormat.Builder.create().setAllowEofWithoutClosingQuote(false).build())) {
- assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
- assertThrows(IOException.class, () -> parser.nextToken(new Token()));
+ assertThrows(IOException.class, () -> lexer.nextToken(new Token()));
}
}
}
[commons-csv] 03/03: Tests CSV-141 and PR 295
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
commit 1269c133ff4637ed658fb8ab5d78a8671ecfed4a
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Sat Jan 21 14:35:28 2023 -0500
Tests CSV-141 and PR 295
---
.../java/org/apache/commons/csv/CSVParserTest.java | 111 +++++++++++++++++++++
.../org/apache/commons/csv/CSV-141/csv-141.csv | 4 +
2 files changed, 115 insertions(+)
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java
index 10c7b97c..851a45d9 100644
--- a/src/test/java/org/apache/commons/csv/CSVParserTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -113,6 +113,14 @@ public class CSVParserTest {
return new BOMInputStream(ClassLoader.getSystemClassLoader().getResource(resource).openStream());
}
+ CSVRecord parse(final CSVParser parser, final int failParseRecordNo) throws IOException {
+ if (parser.getRecordNumber() + 1 == failParseRecordNo) {
+ assertThrows(IOException.class, () -> parser.nextRecord());
+ return null;
+ }
+ return parser.nextRecord();
+ }
+
private void parseFully(final CSVParser parser) {
parser.forEach(Assertions::assertNotNull);
}
@@ -265,6 +273,109 @@ public class CSVParserTest {
assertThrows(NoSuchElementException.class, records::next);
}
+ @Test
+ public void testCSV141_CSVFormat_DEFAULT() throws Exception {
+ testCSV141Failure(CSVFormat.DEFAULT, 3);
+ }
+
+ @Test
+ public void testCSV141CSVFormat_INFORMIX_UNLOAD() throws Exception {
+ testCSV141Failure(CSVFormat.INFORMIX_UNLOAD, 1);
+ }
+
+ @Test
+ public void testCSV141CSVFormat_INFORMIX_UNLOAD_CSV() throws Exception {
+ testCSV141Failure(CSVFormat.INFORMIX_UNLOAD_CSV, 3);
+ }
+
+ @Test
+ public void testCSV141CSVFormat_ORACLE() throws Exception {
+ testCSV141Failure(CSVFormat.ORACLE, 2);
+ }
+
+
+ @Test
+ public void testCSV141CSVFormat_POSTGRESQL_CSV() throws Exception {
+ testCSV141Failure(CSVFormat.POSTGRESQL_CSV, 3);
+ }
+
+ @Test
+ @Disabled("PR 295 does not work")
+ public void testCSV141Excel() throws Exception {
+ testCSV141Ok(CSVFormat.EXCEL);
+ }
+
+ private void testCSV141Failure(final CSVFormat format, final int failParseRecordNo) throws IOException {
+ final Path path = Paths.get("src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv");
+ try (final CSVParser parser = CSVParser.parse(path, StandardCharsets.UTF_8, format)) {
+ // row 1
+ CSVRecord record = parse(parser, failParseRecordNo);
+ if (record == null) {
+ return; // expected failure
+ }
+ assertEquals("1414770317901", record.get(0));
+ assertEquals("android.widget.EditText", record.get(1));
+ assertEquals("pass sem1 _84*|*", record.get(2));
+ assertEquals("0", record.get(3));
+ assertEquals("pass sem1 _8", record.get(4));
+ assertEquals(5, record.size());
+ // row 2
+ record = parse(parser, failParseRecordNo);
+ if (record == null) {
+ return; // expected failure
+ }
+ assertEquals("1414770318470", record.get(0));
+ assertEquals("android.widget.EditText", record.get(1));
+ assertEquals("pass sem1 _84:|", record.get(2));
+ assertEquals("0", record.get(3));
+ assertEquals("pass sem1 _84:\\", record.get(4));
+ assertEquals(5, record.size());
+ // row 3: Fail for certain
+ assertThrows(IOException.class, () -> parser.nextRecord());
+ }
+ }
+
+ private void testCSV141Ok(final CSVFormat format) throws IOException {
+ final Path path = Paths.get("src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv");
+ try (final CSVParser parser = CSVParser.parse(path, StandardCharsets.UTF_8, format)) {
+ // row 1
+ CSVRecord record = parser.nextRecord();
+ assertEquals("1414770317901", record.get(0));
+ assertEquals("android.widget.EditText", record.get(1));
+ assertEquals("pass sem1 _84*|*", record.get(2));
+ assertEquals("0", record.get(3));
+ assertEquals("pass sem1 _8", record.get(4));
+ assertEquals(5, record.size());
+ // row 2
+ record = parser.nextRecord();
+ assertEquals("1414770318470", record.get(0));
+ assertEquals("android.widget.EditText", record.get(1));
+ assertEquals("pass sem1 _84:|", record.get(2));
+ assertEquals("0", record.get(3));
+ assertEquals("pass sem1 _84:\\", record.get(4));
+ assertEquals(5, record.size());
+ // row 3
+ record = parser.nextRecord();
+ assertEquals("1414770318327", record.get(0));
+ assertEquals("android.widget.EditText", record.get(1));
+ assertEquals("pass sem1", record.get(2));
+ assertEquals(3, record.size());
+ // row 4
+ record = parser.nextRecord();
+ assertEquals("1414770318628", record.get(0));
+ assertEquals("android.widget.EditText", record.get(1));
+ assertEquals("pass sem1 _84*|*", record.get(2));
+ assertEquals("0", record.get(3));
+ assertEquals("pass sem1", record.get(4));
+ assertEquals(5, record.size());
+ }
+ }
+
+ @Test
+ public void testCSV141RFC4180() throws Exception {
+ testCSV141Failure(CSVFormat.RFC4180, 3);
+ }
+
@Test
public void testCSV235() throws IOException {
final String dqString = "\"aaa\",\"b\"\"bb\",\"ccc\""; // "aaa","b""bb","ccc"
diff --git a/src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv b/src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv
new file mode 100644
index 00000000..e685adc8
--- /dev/null
+++ b/src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv
@@ -0,0 +1,4 @@
+"1414770317901","android.widget.EditText","pass sem1 _84*|*","0","pass sem1 _8"
+"1414770318470","android.widget.EditText","pass sem1 _84:|","0","pass sem1 _84:\"
+"1414770318327","android.widget.EditText","pass sem1
+"1414770318628","android.widget.EditText","pass sem1 _84*|*","0","pass sem1