You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2024/03/11 19:43:27 UTC
(commons-csv) branch master updated (fca97cd9 -> 7e43e94b)
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
from fca97cd9 [CSV-311] OutOfMemory for very long rows despite using column value of type Reader
new 98f46ac6 Internal refactoring for delimiter
new 50e56c64 Internal refactoring for escape character
new 7e43e94b Whitespace
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/commons/csv/CSVFormat.java | 53 ++++++++++++++++------
.../apache/commons/csv/ExtendedBufferedReader.java | 8 ----
src/main/java/org/apache/commons/csv/Lexer.java | 2 +-
.../java/org/apache/commons/csv/CSVFormatTest.java | 3 ++
4 files changed, 43 insertions(+), 23 deletions(-)
(commons-csv) 02/03: Internal refactoring for escape character
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
commit 50e56c64b6c412d7885ef642b466d21d6aa1be53
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Mon Mar 11 14:08:27 2024 -0400
Internal refactoring for escape character
---
.../java/org/apache/commons/csv/CSVFormat.java | 38 +++++++++++++++-------
.../java/org/apache/commons/csv/CSVFormatTest.java | 3 ++
2 files changed, 30 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/commons/csv/CSVFormat.java b/src/main/java/org/apache/commons/csv/CSVFormat.java
index d7d5ccba..9ab21d5a 100644
--- a/src/main/java/org/apache/commons/csv/CSVFormat.java
+++ b/src/main/java/org/apache/commons/csv/CSVFormat.java
@@ -1717,6 +1717,15 @@ public final class CSVFormat implements Serializable {
return escapeCharacter;
}
+ /**
+ * Gets the escape character.
+ *
+ * @return the escape character, may be {@code 0}
+ */
+ char getEscapeChar() {
+ return escapeCharacter != null ? escapeCharacter.charValue() : 0;
+ }
+
/**
* Gets a copy of the header array.
*
@@ -2129,7 +2138,7 @@ public final class CSVFormat implements Serializable {
}
/*
- * Note: Must only be called if escaping is enabled, otherwise will generate NPE.
+ * Note: Must only be called if escaping is enabled, otherwise can throw exceptions.
*/
private void printWithEscapes(final CharSequence charSeq, final Appendable appendable) throws IOException {
int start = 0;
@@ -2137,18 +2146,20 @@ public final class CSVFormat implements Serializable {
final int end = charSeq.length();
final char[] delim = getDelimiterCharArray();
final int delimLength = delim.length;
- final char escape = getEscapeCharacter().charValue();
+ final char escape = getEscapeChar();
while (pos < end) {
char c = charSeq.charAt(pos);
final boolean isDelimiterStart = isDelimiter(c, charSeq, pos, delim, delimLength);
- if (c == CR || c == LF || c == escape || isDelimiterStart) {
+ final boolean isCr = c == CR;
+ final boolean isLf = c == LF;
+ if (isCr || isLf || c == escape || isDelimiterStart) {
// write out segment up until this char
if (pos > start) {
appendable.append(charSeq, start, pos);
}
- if (c == LF) {
+ if (isLf) {
c = 'n';
- } else if (c == CR) {
+ } else if (isCr) {
c = 'r';
}
appendable.append(escape);
@@ -2172,6 +2183,9 @@ public final class CSVFormat implements Serializable {
}
}
+ /*
+ * Note: Must only be called if escaping is enabled, otherwise can throw exceptions.
+ */
private void printWithEscapes(final Reader reader, final Appendable appendable) throws IOException {
int start = 0;
int pos = 0;
@@ -2179,23 +2193,25 @@ public final class CSVFormat implements Serializable {
final ExtendedBufferedReader bufferedReader = new ExtendedBufferedReader(reader);
final char[] delim = getDelimiterCharArray();
final int delimLength = delim.length;
- final char escape = getEscapeCharacter().charValue();
+ final char escape = getEscapeChar();
final StringBuilder builder = new StringBuilder(IOUtils.DEFAULT_BUFFER_SIZE);
int c;
while (EOF != (c = bufferedReader.read())) {
builder.append((char) c);
final boolean isDelimiterStart = isDelimiter((char) c, builder.toString() + new String(bufferedReader.lookAhead(delimLength - 1)), pos, delim,
delimLength);
- if (c == CR || c == LF || c == escape || isDelimiterStart) {
+ final boolean isCr = c == CR;
+ final boolean isLf = c == LF;
+ if (isCr || isLf || c == escape || isDelimiterStart) {
// write out segment up until this char
if (pos > start) {
append(builder.substring(start, pos), appendable);
builder.setLength(0);
pos = -1;
}
- if (c == LF) {
+ if (isLf) {
c = 'n';
- } else if (c == CR) {
+ } else if (isCr) {
c = 'r';
}
append(escape, appendable);
@@ -2232,7 +2248,7 @@ public final class CSVFormat implements Serializable {
// If escape char not specified, default to the quote char
// This avoids having to keep checking whether there is an escape character
// at the cost of checking against quote twice
- final char escapeChar = isEscapeCharacterSet() ? getEscapeCharacter().charValue() : quoteChar;
+ final char escapeChar = isEscapeCharacterSet() ? getEscapeChar() : quoteChar;
QuoteMode quoteModePolicy = getQuoteMode();
if (quoteModePolicy == null) {
quoteModePolicy = QuoteMode.MINIMAL;
@@ -2436,7 +2452,7 @@ public final class CSVFormat implements Serializable {
final boolean emptyDuplicatesAllowed = duplicateHeaderMode == DuplicateHeaderMode.ALLOW_EMPTY;
for (final String header : headers) {
final boolean blank = isBlank(header);
- // Sanitise all empty headers to the empty string "" when checking duplicates
+ // Sanitize all empty headers to the empty string "" when checking duplicates
final boolean containsHeader = !dupCheckSet.add(blank ? "" : header);
if (containsHeader && !(blank && emptyDuplicatesAllowed)) {
throw new IllegalArgumentException(
diff --git a/src/test/java/org/apache/commons/csv/CSVFormatTest.java b/src/test/java/org/apache/commons/csv/CSVFormatTest.java
index 8bd8fed6..3220759e 100644
--- a/src/test/java/org/apache/commons/csv/CSVFormatTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVFormatTest.java
@@ -393,6 +393,7 @@ public class CSVFormatTest {
final CSVFormat csvFormatTwo = CSVFormat.MYSQL;
assertEquals('\\', (char) csvFormatOne.getEscapeCharacter());
+ assertEquals('\\', csvFormatOne.getEscapeChar());
assertNull(csvFormatOne.getQuoteMode());
assertTrue(csvFormatOne.getIgnoreEmptyLines());
@@ -426,6 +427,8 @@ public class CSVFormatTest {
assertEquals(QuoteMode.ALL_NON_NULL, csvFormatTwo.getQuoteMode());
assertEquals('\t', csvFormatTwo.getDelimiter());
+ assertArrayEquals(new char[] { '\t' }, csvFormatTwo.getDelimiterCharArray());
+ assertEquals("\t", csvFormatTwo.getDelimiterString());
assertEquals("\n", csvFormatTwo.getRecordSeparator());
assertFalse(csvFormatTwo.isQuoteCharacterSet());
(commons-csv) 01/03: Internal refactoring for delimiter
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
commit 98f46ac66169d3dd6b6f3509cc0c426156043913
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Mon Mar 11 13:52:30 2024 -0400
Internal refactoring for delimiter
---
src/main/java/org/apache/commons/csv/CSVFormat.java | 15 ++++++++++++---
src/main/java/org/apache/commons/csv/Lexer.java | 2 +-
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/apache/commons/csv/CSVFormat.java b/src/main/java/org/apache/commons/csv/CSVFormat.java
index 40b2d007..d7d5ccba 100644
--- a/src/main/java/org/apache/commons/csv/CSVFormat.java
+++ b/src/main/java/org/apache/commons/csv/CSVFormat.java
@@ -1679,6 +1679,15 @@ public final class CSVFormat implements Serializable {
return delimiter.charAt(0);
}
+ /**
+ * Gets the character delimiting the values (typically ";", "," or "\t").
+ *
+ * @return the delimiter.
+ */
+ char[] getDelimiterCharArray() {
+ return delimiter.toCharArray();
+ }
+
/**
* Gets the character delimiting the values (typically ";", "," or "\t").
*
@@ -2126,7 +2135,7 @@ public final class CSVFormat implements Serializable {
int start = 0;
int pos = 0;
final int end = charSeq.length();
- final char[] delim = getDelimiterString().toCharArray();
+ final char[] delim = getDelimiterCharArray();
final int delimLength = delim.length;
final char escape = getEscapeCharacter().charValue();
while (pos < end) {
@@ -2168,7 +2177,7 @@ public final class CSVFormat implements Serializable {
int pos = 0;
@SuppressWarnings("resource") // Temp reader on input reader.
final ExtendedBufferedReader bufferedReader = new ExtendedBufferedReader(reader);
- final char[] delim = getDelimiterString().toCharArray();
+ final char[] delim = getDelimiterCharArray();
final int delimLength = delim.length;
final char escape = getEscapeCharacter().charValue();
final StringBuilder builder = new StringBuilder(IOUtils.DEFAULT_BUFFER_SIZE);
@@ -2217,7 +2226,7 @@ public final class CSVFormat implements Serializable {
int start = 0;
int pos = 0;
final int len = charSeq.length();
- final char[] delim = getDelimiterString().toCharArray();
+ final char[] delim = getDelimiterCharArray();
final int delimLength = delim.length;
final char quoteChar = getQuoteCharacter().charValue();
// If escape char not specified, default to the quote char
diff --git a/src/main/java/org/apache/commons/csv/Lexer.java b/src/main/java/org/apache/commons/csv/Lexer.java
index eeb39a64..ef379ec3 100644
--- a/src/main/java/org/apache/commons/csv/Lexer.java
+++ b/src/main/java/org/apache/commons/csv/Lexer.java
@@ -65,7 +65,7 @@ final class Lexer implements Closeable {
Lexer(final CSVFormat format, final ExtendedBufferedReader reader) {
this.reader = reader;
- this.delimiter = format.getDelimiterString().toCharArray();
+ this.delimiter = format.getDelimiterCharArray();
this.escape = mapNullToDisabled(format.getEscapeCharacter());
this.quoteChar = mapNullToDisabled(format.getQuoteCharacter());
this.commentStart = mapNullToDisabled(format.getCommentMarker());
(commons-csv) 03/03: Whitespace
Posted by gg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
commit 7e43e94b4c10d523255c1c7b2be5e6bde264345d
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Mon Mar 11 15:43:22 2024 -0400
Whitespace
---
src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java | 8 --------
1 file changed, 8 deletions(-)
diff --git a/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java b/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java
index 40c7c140..e193c487 100644
--- a/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java
+++ b/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java
@@ -118,7 +118,6 @@ final class ExtendedBufferedReader extends BufferedReader {
super.mark(1);
final int c = super.read();
super.reset();
-
return c;
}
@@ -137,7 +136,6 @@ final class ExtendedBufferedReader extends BufferedReader {
super.mark(n);
super.read(buf, 0, n);
super.reset();
-
return buf;
}
@@ -171,11 +169,8 @@ final class ExtendedBufferedReader extends BufferedReader {
if (length == 0) {
return 0;
}
-
final int len = super.read(buf, offset, length);
-
if (len > 0) {
-
for (int i = offset; i < offset + len; i++) {
final char ch = buf[i];
if (ch == LF) {
@@ -186,13 +181,10 @@ final class ExtendedBufferedReader extends BufferedReader {
eolCounter++;
}
}
-
lastChar = buf[offset + len - 1];
-
} else if (len == EOF) {
lastChar = EOF;
}
-
position += len;
return len;
}