You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2022/09/12 18:19:00 UTC
[commons-csv] branch master updated: Use forEach()
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
The following commit(s) were added to refs/heads/master by this push:
new 379d17c9 Use forEach()
379d17c9 is described below
commit 379d17c983e3a5a89dda3cba8e82df25c5fc7bb0
Author: Gary Gregory <ga...@gmail.com>
AuthorDate: Mon Sep 12 11:18:55 2022 -0700
Use forEach()
---
.../java/org/apache/commons/csv/CSVParserTest.java | 34 +-
.../org/apache/commons/csv/PerformanceTest.java | 658 ++++++++++-----------
.../apache/commons/csv/issues/JiraCsv198Test.java | 108 ++--
.../apache/commons/csv/issues/JiraCsv211Test.java | 109 ++--
.../apache/commons/csv/issues/JiraCsv288Test.java | 4 +-
5 files changed, 445 insertions(+), 468 deletions(-)
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java
index a00803b4..ce26ca7b 100644
--- a/src/test/java/org/apache/commons/csv/CSVParserTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -113,9 +113,7 @@ public class CSVParserTest {
}
private void parseFully(final CSVParser parser) {
- for (final CSVRecord csvRecord : parser) {
- assertNotNull(csvRecord);
- }
+ parser.forEach(record -> assertNotNull(record));
}
@Test
@@ -208,11 +206,7 @@ public class CSVParserTest {
public void testBOM() throws IOException {
final URL url = ClassLoader.getSystemClassLoader().getResource("org/apache/commons/csv/CSVFileParser/bom.csv");
try (final CSVParser parser = CSVParser.parse(url, Charset.forName(UTF_8_NAME), CSVFormat.EXCEL.withHeader())) {
- for (final CSVRecord record : parser) {
- final String string = record.get("Date");
- assertNotNull(string);
- // System.out.println("date: " + record.get("Date"));
- }
+ parser.forEach(record -> assertNotNull(record.get("Date")));
}
}
@@ -220,11 +214,7 @@ public class CSVParserTest {
public void testBOMInputStream_ParserWithInputStream() throws IOException {
try (final BOMInputStream inputStream = createBOMInputStream("org/apache/commons/csv/CSVFileParser/bom.csv");
final CSVParser parser = CSVParser.parse(inputStream, UTF_8, CSVFormat.EXCEL.withHeader())) {
- for (final CSVRecord record : parser) {
- final String string = record.get("Date");
- assertNotNull(string);
- // System.out.println("date: " + record.get("Date"));
- }
+ parser.forEach(record -> assertNotNull(record.get("Date")));
}
}
@@ -232,11 +222,7 @@ public class CSVParserTest {
public void testBOMInputStream_ParserWithReader() throws IOException {
try (final Reader reader = new InputStreamReader(createBOMInputStream("org/apache/commons/csv/CSVFileParser/bom.csv"), UTF_8_NAME);
final CSVParser parser = new CSVParser(reader, CSVFormat.EXCEL.withHeader())) {
- for (final CSVRecord record : parser) {
- final String string = record.get("Date");
- assertNotNull(string);
- // System.out.println("date: " + record.get("Date"));
- }
+ parser.forEach(record -> assertNotNull(record.get("Date")));
}
}
@@ -244,11 +230,7 @@ public class CSVParserTest {
public void testBOMInputStream_parseWithReader() throws IOException {
try (final Reader reader = new InputStreamReader(createBOMInputStream("org/apache/commons/csv/CSVFileParser/bom.csv"), UTF_8_NAME);
final CSVParser parser = CSVParser.parse(reader, CSVFormat.EXCEL.withHeader())) {
- for (final CSVRecord record : parser) {
- final String string = record.get("Date");
- assertNotNull(string);
- // System.out.println("date: " + record.get("Date"));
- }
+ parser.forEach(record -> assertNotNull(record.get("Date")));
}
}
@@ -476,11 +458,11 @@ public class CSVParserTest {
public void testExcelHeaderCountLessThanData() throws Exception {
final String code = "A,B,C,,\r\na,b,c,d,e\r\n";
try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL.withHeader())) {
- for (final CSVRecord record : parser.getRecords()) {
+ parser.getRecords().forEach(record -> {
assertEquals("a", record.get("A"));
assertEquals("b", record.get("B"));
assertEquals("c", record.get("C"));
- }
+ });
}
}
@@ -516,8 +498,8 @@ public class CSVParserTest {
@Test
public void testForEach() throws Exception {
- final List<CSVRecord> records = new ArrayList<>();
try (final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z"); final CSVParser parser = CSVFormat.DEFAULT.parse(in)) {
+ final List<CSVRecord> records = new ArrayList<>();
for (final CSVRecord record : parser) {
records.add(record);
}
diff --git a/src/test/java/org/apache/commons/csv/PerformanceTest.java b/src/test/java/org/apache/commons/csv/PerformanceTest.java
index ea6e8fd1..ef328c63 100644
--- a/src/test/java/org/apache/commons/csv/PerformanceTest.java
+++ b/src/test/java/org/apache/commons/csv/PerformanceTest.java
@@ -1,330 +1,330 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.commons.csv;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.commons.io.IOUtils;
-
-/**
- * Basic test harness.
- */
-@SuppressWarnings("boxing")
-public class PerformanceTest {
-
- @FunctionalInterface
- private interface CSVParserFactory {
- CSVParser createParser() throws IOException;
- }
-
- // Container for basic statistics
- private static class Stats {
- final int count;
- final int fields;
- Stats(final int c, final int f) {
- count = c;
- fields = f;
- }
- }
-
- private static final String[] PROPS = {
- "java.version", // Java Runtime Environment version
- "java.vendor", // Java Runtime Environment vendor
-// "java.vm.specification.version", // Java Virtual Machine specification version
-// "java.vm.specification.vendor", // Java Virtual Machine specification vendor
-// "java.vm.specification.name", // Java Virtual Machine specification name
- "java.vm.version", // Java Virtual Machine implementation version
-// "java.vm.vendor", // Java Virtual Machine implementation vendor
- "java.vm.name", // Java Virtual Machine implementation name
-// "java.specification.version", // Java Runtime Environment specification version
-// "java.specification.vendor", // Java Runtime Environment specification vendor
-// "java.specification.name", // Java Runtime Environment specification name
-
- "os.name", // Operating system name
- "os.arch", // Operating system architecture
- "os.version", // Operating system version
- };
- private static int max = 11; // skip first test
-
- private static int num; // number of elapsed times recorded
-
- private static final long[] ELAPSED_TIMES = new long[max];
- private static final CSVFormat format = CSVFormat.EXCEL;
-
- private static final String TEST_RESRC = "org/apache/commons/csv/perf/worldcitiespop.txt.gz";
-
- private static final File BIG_FILE = new File(System.getProperty("java.io.tmpdir"), "worldcitiespop.txt");
-
- private static Reader createReader() throws IOException {
- return new InputStreamReader(new FileInputStream(BIG_FILE), StandardCharsets.ISO_8859_1);
- }
-
- private static Lexer createTestCSVLexer(final String test, final ExtendedBufferedReader input)
- throws InstantiationException, IllegalAccessException, InvocationTargetException, Exception {
- return test.startsWith("CSVLexer") ? getLexerCtor(test).newInstance(format, input) : new Lexer(format, input);
- }
-
- private static Constructor<Lexer> getLexerCtor(final String clazz) throws Exception {
- @SuppressWarnings("unchecked")
- final Class<Lexer> lexer = (Class<Lexer>) Class.forName("org.apache.commons.csv." + clazz);
- return lexer.getConstructor(CSVFormat.class, ExtendedBufferedReader.class);
- }
-
- private static Stats iterate(final Iterable<CSVRecord> it) {
- int count = 0;
- int fields = 0;
- for (final CSVRecord record : it) {
- count++;
- fields += record.size();
- }
- return new Stats(count, fields);
- }
-
- public static void main(final String [] args) throws Exception {
- if (BIG_FILE.exists()) {
- System.out.printf("Found test fixture %s: %,d bytes.%n", BIG_FILE, BIG_FILE.length());
- } else {
- System.out.println("Decompressing test fixture to: " + BIG_FILE + "...");
- try (
- final InputStream input = new GZIPInputStream(
- PerformanceTest.class.getClassLoader().getResourceAsStream(TEST_RESRC));
- final OutputStream output = new FileOutputStream(BIG_FILE)) {
- IOUtils.copy(input, output);
- System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
- }
- }
- final int argc = args.length;
- if (argc > 0) {
- max = Integer.parseInt(args[0]);
- }
-
- final String[] tests;
- if (argc > 1) {
- tests = new String[argc - 1];
- System.arraycopy(args, 1, tests, 0, argc - 1);
- } else {
- tests = new String[] { "file", "split", "extb", "exts", "csv", "csv-path", "csv-path-db", "csv-url", "lexreset", "lexnew" };
- }
- for (final String p : PROPS) {
- System.out.printf("%s=%s%n", p, System.getProperty(p));
- }
- System.out.printf("Max count: %d%n%n", max);
-
- for (final String test : tests) {
- if ("file".equals(test)) {
- testReadBigFile(false);
- } else if ("split".equals(test)) {
- testReadBigFile(true);
- } else if ("csv".equals(test)) {
- testParseCommonsCSV();
- } else if ("csv-path".equals(test)) {
- testParsePath();
- } else if ("csv-path-db".equals(test)) {
- testParsePathDoubleBuffering();
- } else if ("csv-url".equals(test)) {
- testParseURL();
- } else if ("lexreset".equals(test)) {
- testCSVLexer(false, test);
- } else if ("lexnew".equals(test)) {
- testCSVLexer(true, test);
- } else if (test.startsWith("CSVLexer")) {
- testCSVLexer(false, test);
- } else if ("extb".equals(test)) {
- testExtendedBuffer(false);
- } else if ("exts".equals(test)) {
- testExtendedBuffer(true);
- } else {
- System.out.printf("Invalid test name: %s%n", test);
- }
- }
- }
-
- private static Stats readAll(final BufferedReader in, final boolean split) throws IOException {
- int count = 0;
- int fields = 0;
- String record;
- while ((record = in.readLine()) != null) {
- count++;
- fields += split ? record.split(",").length : 1;
- }
- return new Stats(count, fields);
- }
-
- // calculate and show average
- private static void show(){
- if (num > 1) {
- long tot = 0;
- for (int i = 1; i < num; i++) { // skip first test
- tot += ELAPSED_TIMES[i];
- }
- System.out.printf("%-20s: %5dms%n%n", "Average(not first)", tot / (num - 1));
- }
- num = 0; // ready for next set
- }
-
- // Display end stats; store elapsed for average
- private static void show(final String msg, final Stats s, final long start) {
- final long elapsed = System.currentTimeMillis() - start;
- System.out.printf("%-20s: %5dms %d lines %d fields%n", msg, elapsed, s.count, s.fields);
- ELAPSED_TIMES[num] = elapsed;
- num++;
- }
-
- private static void testCSVLexer(final boolean newToken, final String test) throws Exception {
- Token token = new Token();
- String dynamic = "";
- for (int i = 0; i < max; i++) {
- final String simpleName;
- final Stats stats;
- final long startMillis;
- try (final ExtendedBufferedReader input = new ExtendedBufferedReader(createReader());
- final Lexer lexer = createTestCSVLexer(test, input)) {
- if (test.startsWith("CSVLexer")) {
- dynamic = "!";
- }
- simpleName = lexer.getClass().getSimpleName();
- int count = 0;
- int fields = 0;
- startMillis = System.currentTimeMillis();
- do {
- if (newToken) {
- token = new Token();
- } else {
- token.reset();
- }
- lexer.nextToken(token);
- switch (token.type) {
- case EOF:
- break;
- case EORECORD:
- fields++;
- count++;
- break;
- case INVALID:
- throw new IOException("invalid parse sequence <" + token.content.toString() + ">");
- case TOKEN:
- fields++;
- break;
- case COMMENT: // not really expecting these
- break;
- default:
- throw new IllegalStateException("Unexpected Token type: " + token.type);
- }
- } while (!token.type.equals(Token.Type.EOF));
- stats = new Stats(count, fields);
- }
- show(simpleName + dynamic + " " + (newToken ? "new" : "reset"), stats, startMillis);
- }
- show();
- }
-
- private static void testExtendedBuffer(final boolean makeString) throws Exception {
- for (int i = 0; i < max; i++) {
- int fields = 0;
- int lines = 0;
- final long startMillis;
- try (final ExtendedBufferedReader in = new ExtendedBufferedReader(createReader())) {
- startMillis = System.currentTimeMillis();
- int read;
- if (makeString) {
- StringBuilder sb = new StringBuilder();
- while ((read = in.read()) != -1) {
- sb.append((char) read);
- if (read == ',') { // count delimiters
- sb.toString();
- sb = new StringBuilder();
- fields++;
- } else if (read == '\n') {
- sb.toString();
- sb = new StringBuilder();
- lines++;
- }
- }
- } else {
- while ((read = in.read()) != -1) {
- if (read == ',') { // count delimiters
- fields++;
- } else if (read == '\n') {
- lines++;
- }
- }
- }
- fields += lines; // EOL is a delimiter too
- }
- show("Extended" + (makeString ? " toString" : ""), new Stats(lines, fields), startMillis);
- }
- show();
- }
-
- private static void testParseCommonsCSV() throws Exception {
- testParser("CSV", () -> new CSVParser(createReader(), format));
- }
-
- private static void testParsePath() throws Exception {
- testParser("CSV-PATH", () -> CSVParser.parse(Files.newInputStream(Paths.get(BIG_FILE.toURI())), StandardCharsets.ISO_8859_1, format));
- }
-
- private static void testParsePathDoubleBuffering() throws Exception {
- testParser("CSV-PATH-DB", () -> CSVParser.parse(Files.newBufferedReader(Paths.get(BIG_FILE.toURI()), StandardCharsets.ISO_8859_1), format));
- }
-
- private static void testParser(final String msg, final CSVParserFactory fac) throws Exception {
- for (int i = 0; i < max; i++) {
- final long startMillis;
- final Stats stats;
- try (final CSVParser parser = fac.createParser()) {
- startMillis = System.currentTimeMillis();
- stats = iterate(parser);
- }
- show(msg, stats, startMillis);
- }
- show();
- }
-
- private static void testParseURL() throws Exception {
- testParser("CSV-URL", () -> CSVParser.parse(BIG_FILE.toURI().toURL(), StandardCharsets.ISO_8859_1, format));
- }
-
- private static void testReadBigFile(final boolean split) throws Exception {
- for (int i = 0; i < max; i++) {
- final long startMillis;
- final Stats stats;
- try (final BufferedReader in = new BufferedReader(createReader())) {
- startMillis = System.currentTimeMillis();
- stats = readAll(in, split);
- }
- show(split ? "file+split" : "file", stats, startMillis);
- }
- show();
- }
-
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.csv;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.io.IOUtils;
+
+/**
+ * Basic test harness.
+ */
+@SuppressWarnings("boxing")
+public class PerformanceTest {
+
+ @FunctionalInterface
+ private interface CSVParserFactory {
+ CSVParser createParser() throws IOException;
+ }
+
+ // Container for basic statistics
+ private static class Stats {
+ final int count;
+ final int fields;
+ Stats(final int c, final int f) {
+ count = c;
+ fields = f;
+ }
+ }
+
+ private static final String[] PROPERTY_NAMES = {
+ "java.version", // Java Runtime Environment version
+ "java.vendor", // Java Runtime Environment vendor
+// "java.vm.specification.version", // Java Virtual Machine specification version
+// "java.vm.specification.vendor", // Java Virtual Machine specification vendor
+// "java.vm.specification.name", // Java Virtual Machine specification name
+ "java.vm.version", // Java Virtual Machine implementation version
+// "java.vm.vendor", // Java Virtual Machine implementation vendor
+ "java.vm.name", // Java Virtual Machine implementation name
+// "java.specification.version", // Java Runtime Environment specification version
+// "java.specification.vendor", // Java Runtime Environment specification vendor
+// "java.specification.name", // Java Runtime Environment specification name
+
+ "os.name", // Operating system name
+ "os.arch", // Operating system architecture
+ "os.version", // Operating system version
+ };
+ private static int max = 11; // skip first test
+
+ private static int num; // number of elapsed times recorded
+
+ private static final long[] ELAPSED_TIMES = new long[max];
+ private static final CSVFormat format = CSVFormat.EXCEL;
+
+ private static final String TEST_RESRC = "org/apache/commons/csv/perf/worldcitiespop.txt.gz";
+
+ private static final File BIG_FILE = new File(System.getProperty("java.io.tmpdir"), "worldcitiespop.txt");
+
+ private static Reader createReader() throws IOException {
+ return new InputStreamReader(new FileInputStream(BIG_FILE), StandardCharsets.ISO_8859_1);
+ }
+
+ private static Lexer createTestCSVLexer(final String test, final ExtendedBufferedReader input)
+ throws InstantiationException, IllegalAccessException, InvocationTargetException, Exception {
+ return test.startsWith("CSVLexer") ? getLexerCtor(test).newInstance(format, input) : new Lexer(format, input);
+ }
+
+ private static Constructor<Lexer> getLexerCtor(final String clazz) throws Exception {
+ @SuppressWarnings("unchecked")
+ final Class<Lexer> lexer = (Class<Lexer>) Class.forName("org.apache.commons.csv." + clazz);
+ return lexer.getConstructor(CSVFormat.class, ExtendedBufferedReader.class);
+ }
+
+ private static Stats iterate(final Iterable<CSVRecord> iterable) {
+ int count = 0;
+ int fields = 0;
+ for (final CSVRecord record : iterable) {
+ count++;
+ fields += record.size();
+ }
+ return new Stats(count, fields);
+ }
+
+ public static void main(final String [] args) throws Exception {
+ if (BIG_FILE.exists()) {
+ System.out.printf("Found test fixture %s: %,d bytes.%n", BIG_FILE, BIG_FILE.length());
+ } else {
+ System.out.println("Decompressing test fixture to: " + BIG_FILE + "...");
+ try (
+ final InputStream input = new GZIPInputStream(
+ PerformanceTest.class.getClassLoader().getResourceAsStream(TEST_RESRC));
+ final OutputStream output = new FileOutputStream(BIG_FILE)) {
+ IOUtils.copy(input, output);
+ System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
+ }
+ }
+ final int argc = args.length;
+ if (argc > 0) {
+ max = Integer.parseInt(args[0]);
+ }
+
+ final String[] tests;
+ if (argc > 1) {
+ tests = new String[argc - 1];
+ System.arraycopy(args, 1, tests, 0, argc - 1);
+ } else {
+ tests = new String[] { "file", "split", "extb", "exts", "csv", "csv-path", "csv-path-db", "csv-url", "lexreset", "lexnew" };
+ }
+ for (final String p : PROPERTY_NAMES) {
+ System.out.printf("%s=%s%n", p, System.getProperty(p));
+ }
+ System.out.printf("Max count: %d%n%n", max);
+
+ for (final String test : tests) {
+ if ("file".equals(test)) {
+ testReadBigFile(false);
+ } else if ("split".equals(test)) {
+ testReadBigFile(true);
+ } else if ("csv".equals(test)) {
+ testParseCommonsCSV();
+ } else if ("csv-path".equals(test)) {
+ testParsePath();
+ } else if ("csv-path-db".equals(test)) {
+ testParsePathDoubleBuffering();
+ } else if ("csv-url".equals(test)) {
+ testParseURL();
+ } else if ("lexreset".equals(test)) {
+ testCSVLexer(false, test);
+ } else if ("lexnew".equals(test)) {
+ testCSVLexer(true, test);
+ } else if (test.startsWith("CSVLexer")) {
+ testCSVLexer(false, test);
+ } else if ("extb".equals(test)) {
+ testExtendedBuffer(false);
+ } else if ("exts".equals(test)) {
+ testExtendedBuffer(true);
+ } else {
+ System.out.printf("Invalid test name: %s%n", test);
+ }
+ }
+ }
+
+ private static Stats readAll(final BufferedReader in, final boolean split) throws IOException {
+ int count = 0;
+ int fields = 0;
+ String record;
+ while ((record = in.readLine()) != null) {
+ count++;
+ fields += split ? record.split(",").length : 1;
+ }
+ return new Stats(count, fields);
+ }
+
+ // calculate and show average
+ private static void show(){
+ if (num > 1) {
+ long tot = 0;
+ for (int i = 1; i < num; i++) { // skip first test
+ tot += ELAPSED_TIMES[i];
+ }
+ System.out.printf("%-20s: %5dms%n%n", "Average(not first)", tot / (num - 1));
+ }
+ num = 0; // ready for next set
+ }
+
+ // Display end stats; store elapsed for average
+ private static void show(final String msg, final Stats s, final long start) {
+ final long elapsed = System.currentTimeMillis() - start;
+ System.out.printf("%-20s: %5dms %d lines %d fields%n", msg, elapsed, s.count, s.fields);
+ ELAPSED_TIMES[num] = elapsed;
+ num++;
+ }
+
+ private static void testCSVLexer(final boolean newToken, final String test) throws Exception {
+ Token token = new Token();
+ String dynamic = "";
+ for (int i = 0; i < max; i++) {
+ final String simpleName;
+ final Stats stats;
+ final long startMillis;
+ try (final ExtendedBufferedReader input = new ExtendedBufferedReader(createReader());
+ final Lexer lexer = createTestCSVLexer(test, input)) {
+ if (test.startsWith("CSVLexer")) {
+ dynamic = "!";
+ }
+ simpleName = lexer.getClass().getSimpleName();
+ int count = 0;
+ int fields = 0;
+ startMillis = System.currentTimeMillis();
+ do {
+ if (newToken) {
+ token = new Token();
+ } else {
+ token.reset();
+ }
+ lexer.nextToken(token);
+ switch (token.type) {
+ case EOF:
+ break;
+ case EORECORD:
+ fields++;
+ count++;
+ break;
+ case INVALID:
+ throw new IOException("invalid parse sequence <" + token.content.toString() + ">");
+ case TOKEN:
+ fields++;
+ break;
+ case COMMENT: // not really expecting these
+ break;
+ default:
+ throw new IllegalStateException("Unexpected Token type: " + token.type);
+ }
+ } while (!token.type.equals(Token.Type.EOF));
+ stats = new Stats(count, fields);
+ }
+ show(simpleName + dynamic + " " + (newToken ? "new" : "reset"), stats, startMillis);
+ }
+ show();
+ }
+
+ private static void testExtendedBuffer(final boolean makeString) throws Exception {
+ for (int i = 0; i < max; i++) {
+ int fields = 0;
+ int lines = 0;
+ final long startMillis;
+ try (final ExtendedBufferedReader in = new ExtendedBufferedReader(createReader())) {
+ startMillis = System.currentTimeMillis();
+ int read;
+ if (makeString) {
+ StringBuilder sb = new StringBuilder();
+ while ((read = in.read()) != -1) {
+ sb.append((char) read);
+ if (read == ',') { // count delimiters
+ sb.toString();
+ sb = new StringBuilder();
+ fields++;
+ } else if (read == '\n') {
+ sb.toString();
+ sb = new StringBuilder();
+ lines++;
+ }
+ }
+ } else {
+ while ((read = in.read()) != -1) {
+ if (read == ',') { // count delimiters
+ fields++;
+ } else if (read == '\n') {
+ lines++;
+ }
+ }
+ }
+ fields += lines; // EOL is a delimiter too
+ }
+ show("Extended" + (makeString ? " toString" : ""), new Stats(lines, fields), startMillis);
+ }
+ show();
+ }
+
+ private static void testParseCommonsCSV() throws Exception {
+ testParser("CSV", () -> new CSVParser(createReader(), format));
+ }
+
+ private static void testParsePath() throws Exception {
+ testParser("CSV-PATH", () -> CSVParser.parse(Files.newInputStream(Paths.get(BIG_FILE.toURI())), StandardCharsets.ISO_8859_1, format));
+ }
+
+ private static void testParsePathDoubleBuffering() throws Exception {
+ testParser("CSV-PATH-DB", () -> CSVParser.parse(Files.newBufferedReader(Paths.get(BIG_FILE.toURI()), StandardCharsets.ISO_8859_1), format));
+ }
+
+ private static void testParser(final String msg, final CSVParserFactory fac) throws Exception {
+ for (int i = 0; i < max; i++) {
+ final long startMillis;
+ final Stats stats;
+ try (final CSVParser parser = fac.createParser()) {
+ startMillis = System.currentTimeMillis();
+ stats = iterate(parser);
+ }
+ show(msg, stats, startMillis);
+ }
+ show();
+ }
+
+ private static void testParseURL() throws Exception {
+ testParser("CSV-URL", () -> CSVParser.parse(BIG_FILE.toURI().toURL(), StandardCharsets.ISO_8859_1, format));
+ }
+
+ private static void testReadBigFile(final boolean split) throws Exception {
+ for (int i = 0; i < max; i++) {
+ final long startMillis;
+ final Stats stats;
+ try (final BufferedReader in = new BufferedReader(createReader())) {
+ startMillis = System.currentTimeMillis();
+ stats = readAll(in, split);
+ }
+ show(split ? "file+split" : "file", stats, startMillis);
+ }
+ show();
+ }
+
}
\ No newline at end of file
diff --git a/src/test/java/org/apache/commons/csv/issues/JiraCsv198Test.java b/src/test/java/org/apache/commons/csv/issues/JiraCsv198Test.java
index 60f711ac..1dd072bf 100644
--- a/src/test/java/org/apache/commons/csv/issues/JiraCsv198Test.java
+++ b/src/test/java/org/apache/commons/csv/issues/JiraCsv198Test.java
@@ -1,56 +1,52 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.commons.csv.issues;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVRecord;
-import org.junit.jupiter.api.Test;
-
-public class JiraCsv198Test {
-
- // @formatter:off
- private static final CSVFormat CSV_FORMAT = CSVFormat.EXCEL.builder()
- .setDelimiter('^')
- .setHeader()
- .setSkipHeaderRecord(true)
- .build();
- // @formatter:on
-
- @Test
- public void test() throws UnsupportedEncodingException, IOException {
- final InputStream pointsOfReference = getClass()
- .getResourceAsStream("/org/apache/commons/csv/CSV-198/optd_por_public.csv");
- assertNotNull(pointsOfReference);
- try (@SuppressWarnings("resource")
- CSVParser parser = CSV_FORMAT.parse(new InputStreamReader(pointsOfReference, StandardCharsets.UTF_8))) {
- for (final CSVRecord record : parser) {
- final String locationType = record.get("location_type");
- assertNotNull(locationType);
- }
- }
- }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.csv.issues;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.junit.jupiter.api.Test;
+
+public class JiraCsv198Test {
+
+ // @formatter:off
+ private static final CSVFormat CSV_FORMAT = CSVFormat.EXCEL.builder()
+ .setDelimiter('^')
+ .setHeader()
+ .setSkipHeaderRecord(true)
+ .build();
+ // @formatter:on
+
+ @Test
+ public void test() throws UnsupportedEncodingException, IOException {
+ final InputStream pointsOfReference = getClass().getResourceAsStream("/org/apache/commons/csv/CSV-198/optd_por_public.csv");
+ assertNotNull(pointsOfReference);
+ try (@SuppressWarnings("resource")
+ CSVParser parser = CSV_FORMAT.parse(new InputStreamReader(pointsOfReference, StandardCharsets.UTF_8))) {
+ parser.forEach(record -> assertNotNull(record.get("location_type")));
+ }
+ }
+
+}
diff --git a/src/test/java/org/apache/commons/csv/issues/JiraCsv211Test.java b/src/test/java/org/apache/commons/csv/issues/JiraCsv211Test.java
index 7a9d5ae0..f90f18b0 100644
--- a/src/test/java/org/apache/commons/csv/issues/JiraCsv211Test.java
+++ b/src/test/java/org/apache/commons/csv/issues/JiraCsv211Test.java
@@ -1,55 +1,54 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.commons.csv.issues;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.io.IOException;
-import java.io.StringReader;
-
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVRecord;
-import org.junit.jupiter.api.Test;
-
-public class JiraCsv211Test {
-
- @Test
- public void testJiraCsv211Format() throws IOException {
- final String[] values = {"1", "Jane Doe", "USA", ""};
-
- // @formatter:off
- final CSVFormat printFormat = CSVFormat.DEFAULT.builder()
- .setDelimiter('\t')
- .setHeader("ID", "Name", "Country", "Age")
- .build();
- // @formatter:on
- final String formatted = printFormat.format(values);
- assertEquals("ID\tName\tCountry\tAge\r\n1\tJane Doe\tUSA\t", formatted);
-
- final CSVFormat parseFormat = CSVFormat.DEFAULT.builder().setDelimiter('\t').setHeader()
- .setSkipHeaderRecord(true).build();
- try (final CSVParser parser = parseFormat.parse(new StringReader(formatted))) {
- for (final CSVRecord record : parser) {
- assertEquals("1", record.get(0));
- assertEquals("Jane Doe", record.get(1));
- assertEquals("USA", record.get(2));
- assertEquals("", record.get(3));
- }
- }
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.csv.issues;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.junit.jupiter.api.Test;
+
+public class JiraCsv211Test {
+
+ @Test
+ public void testJiraCsv211Format() throws IOException {
+ final String[] values = {"1", "Jane Doe", "USA", ""};
+
+ // @formatter:off
+ final CSVFormat printFormat = CSVFormat.DEFAULT.builder()
+ .setDelimiter('\t')
+ .setHeader("ID", "Name", "Country", "Age")
+ .build();
+ // @formatter:on
+ final String formatted = printFormat.format(values);
+ assertEquals("ID\tName\tCountry\tAge\r\n1\tJane Doe\tUSA\t", formatted);
+
+ final CSVFormat parseFormat = CSVFormat.DEFAULT.builder().setDelimiter('\t').setHeader().setSkipHeaderRecord(true).build();
+ try (final CSVParser parser = parseFormat.parse(new StringReader(formatted))) {
+ parser.forEach(record -> {
+ assertEquals("1", record.get(0));
+ assertEquals("Jane Doe", record.get(1));
+ assertEquals("USA", record.get(2));
+ assertEquals("", record.get(3));
+ });
+ }
+ }
+}
diff --git a/src/test/java/org/apache/commons/csv/issues/JiraCsv288Test.java b/src/test/java/org/apache/commons/csv/issues/JiraCsv288Test.java
index c78054e7..920dcb73 100644
--- a/src/test/java/org/apache/commons/csv/issues/JiraCsv288Test.java
+++ b/src/test/java/org/apache/commons/csv/issues/JiraCsv288Test.java
@@ -36,8 +36,8 @@ public class JiraCsv288Test {
final Reader in = new StringReader("a|~|b|~|c|~|d|~||~|f");
final StringBuilder stringBuilder = new StringBuilder();
try (CSVPrinter csvPrinter = new CSVPrinter(stringBuilder, CSVFormat.EXCEL);
- CSVParser csvParser = CSVParser.parse(in, CSVFormat.Builder.create().setDelimiter("|~|").build())) {
- for (final CSVRecord csvRecord : csvParser) {
+ CSVParser parser = CSVParser.parse(in, CSVFormat.Builder.create().setDelimiter("|~|").build())) {
+ for (final CSVRecord csvRecord : parser) {
for (int i = 0; i < csvRecord.size(); i++) {
csvPrinter.print(csvRecord.get(i));
}