You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by jo...@apache.org on 2020/01/18 02:39:06 UTC
[druid] branch 0.17.0 updated: Fix TSV bugs (#9199) (#9213)
This is an automated email from the ASF dual-hosted git repository.
jonwei pushed a commit to branch 0.17.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/0.17.0 by this push:
new 6074dc2 Fix TSV bugs (#9199) (#9213)
6074dc2 is described below
commit 6074dc2ab0ee728073bf6b409535e27b17f7449f
Author: Jihoon Son <ji...@apache.org>
AuthorDate: Fri Jan 17 18:38:57 2020 -0800
Fix TSV bugs (#9199) (#9213)
* working
* - support multi-char delimiter for tsv
- respect "delimiter" property for tsv
* default value check for findColumnsFromHeader
* remove CSVParser to have a true and only CSVParser
* fix tests
* fix another test
---
.../org/apache/druid/data/input/TextReader.java | 7 +-
.../druid/data/input/impl/CsvInputFormat.java | 53 +++++-
.../data/input/impl/DelimitedInputFormat.java | 186 +++++----------------
.../data/input/impl/DelimitedValueReader.java | 48 +-----
.../druid/data/input/impl/FlatTextInputFormat.java | 140 ++++++++++++++++
.../druid/data/input/impl/JsonInputFormat.java | 2 +-
.../apache/druid/data/input/impl/JsonReader.java | 4 +-
.../druid/data/input/impl/RegexInputFormat.java | 2 +-
.../apache/druid/data/input/impl/RegexReader.java | 4 +-
.../druid/java/util/common/parsers/CSVParser.java | 16 +-
.../java/util/common/parsers/DelimitedParser.java | 37 +---
.../druid/data/input/impl/CsvInputFormatTest.java | 46 ++++-
.../data/input/impl/DelimitedInputFormatTest.java | 79 +++++++--
.../druid/data/input/impl/DelimitedReaderTest.java | 101 +----------
14 files changed, 376 insertions(+), 349 deletions(-)
diff --git a/core/src/main/java/org/apache/druid/data/input/TextReader.java b/core/src/main/java/org/apache/druid/data/input/TextReader.java
index 33151bd..7214d32 100644
--- a/core/src/main/java/org/apache/druid/data/input/TextReader.java
+++ b/core/src/main/java/org/apache/druid/data/input/TextReader.java
@@ -26,7 +26,6 @@ import org.apache.druid.java.util.common.parsers.CloseableIterator;
import org.apache.druid.java.util.common.parsers.ParseException;
import org.apache.druid.java.util.common.parsers.ParserUtils;
-import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
@@ -38,14 +37,12 @@ import java.util.List;
public abstract class TextReader extends IntermediateRowParsingReader<String>
{
private final InputRowSchema inputRowSchema;
- final InputEntity source;
- final File temporaryDirectory;
+ private final InputEntity source;
- public TextReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
+ public TextReader(InputRowSchema inputRowSchema, InputEntity source)
{
this.inputRowSchema = inputRowSchema;
this.source = source;
- this.temporaryDirectory = temporaryDirectory;
}
public InputRowSchema getInputRowSchema()
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/CsvInputFormat.java b/core/src/main/java/org/apache/druid/data/input/impl/CsvInputFormat.java
index fe9f438..6d2136b 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/CsvInputFormat.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/CsvInputFormat.java
@@ -20,13 +20,26 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.opencsv.RFC4180Parser;
+import com.opencsv.RFC4180ParserBuilder;
+import com.opencsv.enums.CSVReaderNullFieldIndicator;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.data.input.InputEntity;
+import org.apache.druid.data.input.InputEntityReader;
+import org.apache.druid.data.input.InputRowSchema;
import javax.annotation.Nullable;
+import java.io.File;
+import java.util.Arrays;
import java.util.List;
-public class CsvInputFormat extends DelimitedInputFormat
+public class CsvInputFormat extends FlatTextInputFormat
{
+ private static final char SEPARATOR = ',';
+ private static final RFC4180Parser PARSER = createOpenCsvParser();
+
@JsonCreator
public CsvInputFormat(
@JsonProperty("columns") @Nullable List<String> columns,
@@ -36,6 +49,42 @@ public class CsvInputFormat extends DelimitedInputFormat
@JsonProperty("skipHeaderRows") int skipHeaderRows
)
{
- super(columns, listDelimiter, ",", hasHeaderRow, findColumnsFromHeader, skipHeaderRows);
+ super(columns, listDelimiter, String.valueOf(SEPARATOR), hasHeaderRow, findColumnsFromHeader, skipHeaderRows);
+ }
+
+ @Override
+ @JsonIgnore
+ public String getDelimiter()
+ {
+ return super.getDelimiter();
+ }
+
+ @Override
+ public boolean isSplittable()
+ {
+ return true;
+ }
+
+ @Override
+ public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
+ {
+ return new DelimitedValueReader(
+ inputRowSchema,
+ source,
+ getListDelimiter(),
+ getColumns(),
+ isFindColumnsFromHeader(),
+ getSkipHeaderRows(),
+ line -> Arrays.asList(PARSER.parseLine(line))
+ );
+ }
+
+ public static RFC4180Parser createOpenCsvParser()
+ {
+ return NullHandling.replaceWithDefault()
+ ? new RFC4180ParserBuilder().withSeparator(SEPARATOR).build()
+ : new RFC4180ParserBuilder().withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS)
+ .withSeparator(SEPARATOR)
+ .build();
}
}
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/DelimitedInputFormat.java b/core/src/main/java/org/apache/druid/data/input/impl/DelimitedInputFormat.java
index 7d6bc01..8562cde 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/DelimitedInputFormat.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/DelimitedInputFormat.java
@@ -19,72 +19,30 @@
package org.apache.druid.data.input.impl;
+import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
+import com.google.common.base.Splitter;
+import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.InputEntity;
import org.apache.druid.data.input.InputEntityReader;
-import org.apache.druid.data.input.InputFormat;
import org.apache.druid.data.input.InputRowSchema;
-import org.apache.druid.indexer.Checks;
-import org.apache.druid.indexer.Property;
import javax.annotation.Nullable;
import java.io.File;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
-import java.util.Objects;
/**
* InputFormat for customized Delimitor Separate Value format of input data(default is TSV).
*/
-public class DelimitedInputFormat implements InputFormat
+public class DelimitedInputFormat extends FlatTextInputFormat
{
+ private static final String DEFAULT_DELIMITER = "\t";
- public enum Format
- {
- CSV(',', "comma"),
- TSV('\t', "tab"),
- CustomizeSV('|', "");
-
- private char delimiter;
- private String literal;
-
- Format(char delimiter, String literal)
- {
- this.delimiter = delimiter;
- this.literal = literal;
- }
-
- public String getDelimiterAsString()
- {
- return String.valueOf(delimiter);
- }
-
- private void setDelimiter(String delimiter, String literal)
- {
- this.delimiter = (delimiter != null && delimiter.length() > 0) ? delimiter.charAt(0) : '\t';
- this.literal = literal != null ? literal : "customize separator: " + delimiter;
- }
-
- public char getDelimiter()
- {
- return delimiter;
- }
-
- public String getLiteral()
- {
- return literal;
- }
- }
-
- private final String listDelimiter;
- private final List<String> columns;
- private final boolean findColumnsFromHeader;
- private final int skipHeaderRows;
- private final Format format;
- private final String delimiter;
-
+ @JsonCreator
public DelimitedInputFormat(
@JsonProperty("columns") @Nullable List<String> columns,
@JsonProperty("listDelimiter") @Nullable String listDelimiter,
@@ -94,79 +52,14 @@ public class DelimitedInputFormat implements InputFormat
@JsonProperty("skipHeaderRows") int skipHeaderRows
)
{
- this.listDelimiter = listDelimiter;
- this.columns = columns == null ? Collections.emptyList() : columns;
- //noinspection ConstantConditions
- this.findColumnsFromHeader = Checks.checkOneNotNullOrEmpty(
- ImmutableList.of(
- new Property<>("hasHeaderRow", hasHeaderRow),
- new Property<>("findColumnsFromHeader", findColumnsFromHeader)
- )
- ).getValue();
- this.skipHeaderRows = skipHeaderRows;
- this.delimiter = delimiter == null ? "\t" : delimiter;
- this.format = getFormat(this.delimiter);
- Preconditions.checkArgument(
- this.delimiter.length() == 1,
- "The delimiter should be a single character"
- );
- Preconditions.checkArgument(
- !this.delimiter.equals(listDelimiter),
- "Cannot have same delimiter and list delimiter of [%s]",
- this.delimiter
+ super(
+ columns,
+ listDelimiter,
+ delimiter == null ? DEFAULT_DELIMITER : delimiter,
+ hasHeaderRow,
+ findColumnsFromHeader,
+ skipHeaderRows
);
- if (!this.columns.isEmpty()) {
- for (String column : this.columns) {
- Preconditions.checkArgument(
- !column.contains(format.getDelimiterAsString()),
- "Column[%s] has a " + format.getLiteral() + ", it cannot",
- column
- );
- }
- } else {
- Preconditions.checkArgument(
- this.findColumnsFromHeader,
- "If columns field is not set, the first row of your data must have your header"
- + " and hasHeaderRow must be set to true."
- );
- }
- }
-
- private static Format getFormat(String delimiter)
- {
- if (",".equals(delimiter)) {
- return Format.CSV;
- } else if ("\t".equals(delimiter)) {
- return Format.TSV;
- } else {
- Format.CustomizeSV.setDelimiter(delimiter, null);
- return Format.CustomizeSV;
- }
- }
-
-
- @JsonProperty
- public List<String> getColumns()
- {
- return columns;
- }
-
- @JsonProperty
- public String getListDelimiter()
- {
- return listDelimiter;
- }
-
- @JsonProperty
- public boolean isFindColumnsFromHeader()
- {
- return findColumnsFromHeader;
- }
-
- @JsonProperty
- public int getSkipHeaderRows()
- {
- return skipHeaderRows;
}
@Override
@@ -181,35 +74,36 @@ public class DelimitedInputFormat implements InputFormat
return new DelimitedValueReader(
inputRowSchema,
source,
- temporaryDirectory,
- listDelimiter,
- columns,
- findColumnsFromHeader,
- skipHeaderRows,
- this.format
+ getListDelimiter(),
+ getColumns(),
+ isFindColumnsFromHeader(),
+ getSkipHeaderRows(),
+ line -> splitToList(Splitter.on(getDelimiter()), line)
);
}
- @Override
- public boolean equals(Object o)
+ /**
+ * Copied from Guava's {@link Splitter#splitToList(CharSequence)}.
+ * This is to avoid the error of the missing method signature when using an old Guava library.
+ * For example, it may happen when running Druid Hadoop indexing jobs, since we may inherit the version provided by
+ * the Hadoop cluster. See https://github.com/apache/druid/issues/6801.
+ */
+ public static List<String> splitToList(Splitter splitter, String input)
{
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
+ Preconditions.checkNotNull(input);
+
+ Iterator<String> iterator = splitter.split(input).iterator();
+ List<String> result = new ArrayList<>();
+
+ while (iterator.hasNext()) {
+ String splitValue = iterator.next();
+ if (!NullHandling.replaceWithDefault() && splitValue.isEmpty()) {
+ result.add(null);
+ } else {
+ result.add(splitValue);
+ }
}
- DelimitedInputFormat format = (DelimitedInputFormat) o;
- return findColumnsFromHeader == format.findColumnsFromHeader &&
- skipHeaderRows == format.skipHeaderRows &&
- Objects.equals(listDelimiter, format.listDelimiter) &&
- Objects.equals(columns, format.columns) &&
- Objects.equals(this.format, format.format);
- }
- @Override
- public int hashCode()
- {
- return Objects.hash(listDelimiter, columns, findColumnsFromHeader, skipHeaderRows, format);
+ return Collections.unmodifiableList(result);
}
}
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/DelimitedValueReader.java b/core/src/main/java/org/apache/druid/data/input/impl/DelimitedValueReader.java
index cae21b1..b8b524e 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/DelimitedValueReader.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/DelimitedValueReader.java
@@ -23,10 +23,6 @@ import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
-import com.opencsv.RFC4180Parser;
-import com.opencsv.RFC4180ParserBuilder;
-import com.opencsv.enums.CSVReaderNullFieldIndicator;
-import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.InputEntity;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.InputRowSchema;
@@ -38,9 +34,7 @@ import org.apache.druid.java.util.common.parsers.ParserUtils;
import org.apache.druid.java.util.common.parsers.Parsers;
import javax.annotation.Nullable;
-import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -53,56 +47,32 @@ public class DelimitedValueReader extends TextReader
private final boolean findColumnsFromHeader;
private final int skipHeaderRows;
private final Function<String, Object> multiValueFunction;
+ private final DelimitedValueParser parser;
@Nullable
private List<String> columns;
- private final RFC4180Parser parser;
- public static RFC4180Parser createOpenCsvParser(char separator)
+ interface DelimitedValueParser
{
- return NullHandling.replaceWithDefault()
- ? new RFC4180ParserBuilder()
- .withSeparator(separator)
- .build()
- : new RFC4180ParserBuilder().withFieldAsNull(
- CSVReaderNullFieldIndicator.EMPTY_SEPARATORS)
- .withSeparator(separator)
- .build();
+ List<String> parseLine(String line) throws IOException;
}
DelimitedValueReader(
InputRowSchema inputRowSchema,
InputEntity source,
- File temporaryDirectory,
@Nullable String listDelimiter,
@Nullable List<String> columns,
boolean findColumnsFromHeader,
int skipHeaderRows,
- DelimitedInputFormat.Format format
+ DelimitedValueParser parser
)
{
- super(inputRowSchema, source, temporaryDirectory);
+ super(inputRowSchema, source);
this.findColumnsFromHeader = findColumnsFromHeader;
this.skipHeaderRows = skipHeaderRows;
final String finalListDelimeter = listDelimiter == null ? Parsers.DEFAULT_LIST_DELIMITER : listDelimiter;
this.multiValueFunction = ParserUtils.getMultiValueFunction(finalListDelimeter, Splitter.on(finalListDelimeter));
this.columns = findColumnsFromHeader ? null : columns; // columns will be overriden by header row
- this.parser = createOpenCsvParser(format.getDelimiter());
-
- if (this.columns != null) {
- for (String column : this.columns) {
- Preconditions.checkArgument(
- !column.contains(format.getDelimiterAsString()),
- "Column[%s] has a " + format.getLiteral() + ", it cannot",
- column
- );
- }
- } else {
- Preconditions.checkArgument(
- findColumnsFromHeader,
- "If columns field is not set, the first row of your data must have your header"
- + " and hasHeaderRow must be set to true."
- );
- }
+ this.parser = parser;
}
@Override
@@ -120,10 +90,10 @@ public class DelimitedValueReader extends TextReader
private Map<String, Object> parseLine(String line) throws IOException
{
- final String[] parsed = parser.parseLine(line);
+ final List<String> parsed = parser.parseLine(line);
return Utils.zipMapPartial(
Preconditions.checkNotNull(columns, "columns"),
- Iterables.transform(Arrays.asList(parsed), multiValueFunction)
+ Iterables.transform(parsed, multiValueFunction)
);
}
@@ -145,7 +115,7 @@ public class DelimitedValueReader extends TextReader
if (!findColumnsFromHeader) {
throw new ISE("Don't call this if findColumnsFromHeader = false");
}
- columns = findOrCreateColumnNames(Arrays.asList(parser.parseLine(line)));
+ columns = findOrCreateColumnNames(parser.parseLine(line));
if (columns.isEmpty()) {
throw new ISE("Empty columns");
}
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/FlatTextInputFormat.java b/core/src/main/java/org/apache/druid/data/input/impl/FlatTextInputFormat.java
new file mode 100644
index 0000000..2fc9d7f
--- /dev/null
+++ b/core/src/main/java/org/apache/druid/data/input/impl/FlatTextInputFormat.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.data.input.impl;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.data.input.InputFormat;
+import org.apache.druid.indexer.Checks;
+import org.apache.druid.indexer.Property;
+
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+public abstract class FlatTextInputFormat implements InputFormat
+{
+ private final List<String> columns;
+ private final String listDelimiter;
+ private final String delimiter;
+ private final boolean findColumnsFromHeader;
+ private final int skipHeaderRows;
+
+ FlatTextInputFormat(
+ @Nullable List<String> columns,
+ @Nullable String listDelimiter,
+ String delimiter,
+ @Nullable Boolean hasHeaderRow,
+ @Nullable Boolean findColumnsFromHeader,
+ int skipHeaderRows
+ )
+ {
+ this.columns = columns == null ? Collections.emptyList() : columns;
+ this.listDelimiter = listDelimiter;
+ this.delimiter = Preconditions.checkNotNull(delimiter, "delimiter");
+ //noinspection ConstantConditions
+ if (columns == null || columns.isEmpty()) {
+ this.findColumnsFromHeader = Checks.checkOneNotNullOrEmpty(
+ ImmutableList.of(
+ new Property<>("hasHeaderRow", hasHeaderRow),
+ new Property<>("findColumnsFromHeader", findColumnsFromHeader)
+ )
+ ).getValue();
+ } else {
+ this.findColumnsFromHeader = findColumnsFromHeader == null ? false : findColumnsFromHeader;
+ }
+ this.skipHeaderRows = skipHeaderRows;
+ Preconditions.checkArgument(
+ !delimiter.equals(listDelimiter),
+ "Cannot have same delimiter and list delimiter of [%s]",
+ delimiter
+ );
+ if (!this.columns.isEmpty()) {
+ for (String column : this.columns) {
+ Preconditions.checkArgument(
+ !column.contains(delimiter),
+ "Column[%s] cannot have the delimiter[" + delimiter + "] in its name",
+ column
+ );
+ }
+ } else {
+ Preconditions.checkArgument(
+ this.findColumnsFromHeader,
+ "If columns field is not set, the first row of your data must have your header"
+ + " and hasHeaderRow must be set to true."
+ );
+ }
+ }
+
+ @JsonProperty
+ public String getDelimiter()
+ {
+ return delimiter;
+ }
+
+ @JsonProperty
+ public List<String> getColumns()
+ {
+ return columns;
+ }
+
+ @JsonProperty
+ public String getListDelimiter()
+ {
+ return listDelimiter;
+ }
+
+ @JsonProperty
+ public boolean isFindColumnsFromHeader()
+ {
+ return findColumnsFromHeader;
+ }
+
+ @JsonProperty
+ public int getSkipHeaderRows()
+ {
+ return skipHeaderRows;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ FlatTextInputFormat that = (FlatTextInputFormat) o;
+ return findColumnsFromHeader == that.findColumnsFromHeader &&
+ skipHeaderRows == that.skipHeaderRows &&
+ Objects.equals(listDelimiter, that.listDelimiter) &&
+ Objects.equals(columns, that.columns) &&
+ Objects.equals(delimiter, that.delimiter);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(listDelimiter, columns, findColumnsFromHeader, skipHeaderRows, delimiter);
+ }
+}
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/JsonInputFormat.java b/core/src/main/java/org/apache/druid/data/input/impl/JsonInputFormat.java
index 0cf8a9f..5026bab 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/JsonInputFormat.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/JsonInputFormat.java
@@ -70,7 +70,7 @@ public class JsonInputFormat extends NestedInputFormat
@Override
public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
{
- return new JsonReader(inputRowSchema, source, temporaryDirectory, getFlattenSpec(), objectMapper);
+ return new JsonReader(inputRowSchema, source, getFlattenSpec(), objectMapper);
}
@Override
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/JsonReader.java b/core/src/main/java/org/apache/druid/data/input/impl/JsonReader.java
index 50617fc..f1e8cce 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/JsonReader.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/JsonReader.java
@@ -31,7 +31,6 @@ import org.apache.druid.java.util.common.parsers.ObjectFlattener;
import org.apache.druid.java.util.common.parsers.ObjectFlatteners;
import org.apache.druid.java.util.common.parsers.ParseException;
-import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@@ -45,12 +44,11 @@ public class JsonReader extends TextReader
JsonReader(
InputRowSchema inputRowSchema,
InputEntity source,
- File temporaryDirectory,
JSONPathSpec flattenSpec,
ObjectMapper mapper
)
{
- super(inputRowSchema, source, temporaryDirectory);
+ super(inputRowSchema, source);
this.flattener = ObjectFlatteners.create(flattenSpec, new JSONFlattenerMaker());
this.mapper = mapper;
}
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/RegexInputFormat.java b/core/src/main/java/org/apache/druid/data/input/impl/RegexInputFormat.java
index 239ee63..1165516 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/RegexInputFormat.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/RegexInputFormat.java
@@ -57,6 +57,6 @@ public class RegexInputFormat implements InputFormat
@Override
public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
{
- return new RegexReader(inputRowSchema, source, temporaryDirectory, pattern, listDelimiter, columns);
+ return new RegexReader(inputRowSchema, source, pattern, listDelimiter, columns);
}
}
diff --git a/core/src/main/java/org/apache/druid/data/input/impl/RegexReader.java b/core/src/main/java/org/apache/druid/data/input/impl/RegexReader.java
index c7105c8..2962ebd 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/RegexReader.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/RegexReader.java
@@ -32,7 +32,6 @@ import org.apache.druid.java.util.common.parsers.ParserUtils;
import org.apache.druid.java.util.common.parsers.Parsers;
import javax.annotation.Nullable;
-import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -51,13 +50,12 @@ public class RegexReader extends TextReader
RegexReader(
InputRowSchema inputRowSchema,
InputEntity source,
- File temporaryDirectory,
String pattern,
@Nullable String listDelimiter,
@Nullable List<String> columns
)
{
- super(inputRowSchema, source, temporaryDirectory);
+ super(inputRowSchema, source);
this.pattern = pattern;
this.compiled = Pattern.compile(pattern);
final String finalListDelimeter = listDelimiter == null ? Parsers.DEFAULT_LIST_DELIMITER : listDelimiter;
diff --git a/core/src/main/java/org/apache/druid/java/util/common/parsers/CSVParser.java b/core/src/main/java/org/apache/druid/java/util/common/parsers/CSVParser.java
index e6da347..49f53da 100644
--- a/core/src/main/java/org/apache/druid/java/util/common/parsers/CSVParser.java
+++ b/core/src/main/java/org/apache/druid/java/util/common/parsers/CSVParser.java
@@ -21,7 +21,7 @@ package org.apache.druid.java.util.common.parsers;
import com.google.common.annotations.VisibleForTesting;
import com.opencsv.RFC4180Parser;
-import org.apache.druid.data.input.impl.DelimitedValueReader;
+import org.apache.druid.data.input.impl.CsvInputFormat;
import javax.annotation.Nullable;
import java.io.IOException;
@@ -30,7 +30,7 @@ import java.util.List;
public class CSVParser extends AbstractFlatTextFormatParser
{
- private final RFC4180Parser parser = DelimitedValueReader.createOpenCsvParser(',');
+ private final RFC4180Parser parser = CsvInputFormat.createOpenCsvParser();
public CSVParser(
@Nullable final String listDelimiter,
@@ -53,12 +53,6 @@ public class CSVParser extends AbstractFlatTextFormatParser
setFieldNames(fieldNames);
}
- @Override
- protected List<String> parseLine(String input) throws IOException
- {
- return Arrays.asList(parser.parseLine(input));
- }
-
@VisibleForTesting
CSVParser(@Nullable final String listDelimiter, final String header)
{
@@ -66,4 +60,10 @@ public class CSVParser extends AbstractFlatTextFormatParser
setFieldNames(header);
}
+
+ @Override
+ protected List<String> parseLine(String input) throws IOException
+ {
+ return Arrays.asList(parser.parseLine(input));
+ }
}
diff --git a/core/src/main/java/org/apache/druid/java/util/common/parsers/DelimitedParser.java b/core/src/main/java/org/apache/druid/java/util/common/parsers/DelimitedParser.java
index 0c0dccc..df75cc3 100644
--- a/core/src/main/java/org/apache/druid/java/util/common/parsers/DelimitedParser.java
+++ b/core/src/main/java/org/apache/druid/java/util/common/parsers/DelimitedParser.java
@@ -22,17 +22,13 @@ package org.apache.druid.java.util.common.parsers;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
-import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.data.input.impl.DelimitedInputFormat;
import javax.annotation.Nullable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
public class DelimitedParser extends AbstractFlatTextFormatParser
{
- private final String delimiter;
private final Splitter splitter;
public DelimitedParser(
@@ -43,15 +39,15 @@ public class DelimitedParser extends AbstractFlatTextFormatParser
)
{
super(listDelimiter, hasHeaderRow, maxSkipHeaderRows);
- this.delimiter = delimiter != null ? delimiter : FlatTextFormat.DELIMITED.getDefaultDelimiter();
+ final String finalDelimiter = delimiter != null ? delimiter : FlatTextFormat.DELIMITED.getDefaultDelimiter();
Preconditions.checkState(
- !this.delimiter.equals(getListDelimiter()),
+ !finalDelimiter.equals(getListDelimiter()),
"Cannot have same delimiter and list delimiter of [%s]",
- this.delimiter
+ finalDelimiter
);
- this.splitter = Splitter.on(this.delimiter);
+ this.splitter = Splitter.on(finalDelimiter);
}
public DelimitedParser(
@@ -81,28 +77,9 @@ public class DelimitedParser extends AbstractFlatTextFormatParser
return splitToList(input);
}
- /**
- * Copied from Guava's {@link Splitter#splitToList(CharSequence)}.
- * This is to avoid the error of the missing method signature when using an old Guava library.
- * For example, it may happen when running Druid Hadoop indexing jobs, since we may inherit the version provided by
- * the Hadoop cluster. See https://github.com/apache/druid/issues/6801.
- */
+
private List<String> splitToList(String input)
{
- Preconditions.checkNotNull(input);
-
- Iterator<String> iterator = splitter.split(input).iterator();
- List<String> result = new ArrayList<String>();
-
- while (iterator.hasNext()) {
- String splitValue = iterator.next();
- if (!NullHandling.replaceWithDefault() && splitValue.isEmpty()) {
- result.add(null);
- } else {
- result.add(splitValue);
- }
- }
-
- return Collections.unmodifiableList(result);
+ return DelimitedInputFormat.splitToList(splitter, input);
}
}
diff --git a/core/src/test/java/org/apache/druid/data/input/impl/CsvInputFormatTest.java b/core/src/test/java/org/apache/druid/data/input/impl/CsvInputFormatTest.java
index ad2316c..299fb14 100644
--- a/core/src/test/java/org/apache/druid/data/input/impl/CsvInputFormatTest.java
+++ b/core/src/test/java/org/apache/druid/data/input/impl/CsvInputFormatTest.java
@@ -48,7 +48,7 @@ public class CsvInputFormatTest
public void testComma()
{
expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Column[a,] has a comma, it cannot");
+ expectedException.expectMessage("Column[a,] cannot have the delimiter[,] in its name");
new CsvInputFormat(Collections.singletonList("a,"), "|", null, false, 0);
}
@@ -59,4 +59,48 @@ public class CsvInputFormatTest
expectedException.expectMessage("Cannot have same delimiter and list delimiter of [,]");
new CsvInputFormat(Collections.singletonList("a\t"), ",", null, false, 0);
}
+
+ @Test
+ public void testFindColumnsFromHeaderWithColumnsReturningItsValue()
+ {
+ final CsvInputFormat format = new CsvInputFormat(Collections.singletonList("a"), null, null, true, 0);
+ Assert.assertTrue(format.isFindColumnsFromHeader());
+ }
+
+ @Test
+ public void testFindColumnsFromHeaderWithMissingColumnsReturningItsValue()
+ {
+ final CsvInputFormat format = new CsvInputFormat(null, null, null, true, 0);
+ Assert.assertTrue(format.isFindColumnsFromHeader());
+ }
+
+ @Test
+ public void testMissingFindColumnsFromHeaderWithMissingColumnsThrowingError()
+ {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("At least one of [Property{name='hasHeaderRow', value=null}");
+ new CsvInputFormat(null, null, null, null, 0);
+ }
+
+ @Test
+ public void testMissingFindColumnsFromHeaderWithColumnsReturningFalse()
+ {
+ final CsvInputFormat format = new CsvInputFormat(Collections.singletonList("a"), null, null, null, 0);
+ Assert.assertFalse(format.isFindColumnsFromHeader());
+ }
+
+ @Test
+ public void testHasHeaderRowWithMissingFindColumnsThrowingError()
+ {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("At most one of [Property{name='hasHeaderRow', value=true}");
+ new CsvInputFormat(null, null, true, false, 0);
+ }
+
+ @Test
+ public void testHasHeaderRowWithMissingColumnsReturningItsValue()
+ {
+ final CsvInputFormat format = new CsvInputFormat(null, null, true, null, 0);
+ Assert.assertTrue(format.isFindColumnsFromHeader());
+ }
}
diff --git a/core/src/test/java/org/apache/druid/data/input/impl/DelimitedInputFormatTest.java b/core/src/test/java/org/apache/druid/data/input/impl/DelimitedInputFormatTest.java
index af06d8a..a21e34d 100644
--- a/core/src/test/java/org/apache/druid/data/input/impl/DelimitedInputFormatTest.java
+++ b/core/src/test/java/org/apache/druid/data/input/impl/DelimitedInputFormatTest.java
@@ -41,7 +41,7 @@ public class DelimitedInputFormatTest
final DelimitedInputFormat format = new DelimitedInputFormat(
Collections.singletonList("a"),
"|",
- null,
+ "delim",
null,
true,
10
@@ -55,19 +55,11 @@ public class DelimitedInputFormatTest
public void testTab()
{
expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Column[a\t] has a tab, it cannot");
+ expectedException.expectMessage("Column[a\t] cannot have the delimiter[\t] in its name");
new DelimitedInputFormat(Collections.singletonList("a\t"), ",", null, null, false, 0);
}
@Test
- public void testDelimiterLength()
- {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("The delimiter should be a single character");
- new DelimitedInputFormat(Collections.singletonList("a\t"), ",", "null", null, false, 0);
- }
-
- @Test
public void testDelimiterAndListDelimiter()
{
expectedException.expect(IllegalArgumentException.class);
@@ -79,7 +71,72 @@ public class DelimitedInputFormatTest
public void testCustomizeSeparator()
{
expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("Column[a|] has a customize separator: |, it cannot");
+ expectedException.expectMessage("Column[a|] cannot have the delimiter[|] in its name");
new DelimitedInputFormat(Collections.singletonList("a|"), ",", "|", null, false, 0);
}
+
+ @Test
+ public void testFindColumnsFromHeaderWithColumnsReturningItsValue()
+ {
+ final DelimitedInputFormat format = new DelimitedInputFormat(
+ Collections.singletonList("a"),
+ null,
+ "delim",
+ null,
+ true,
+ 0
+ );
+ Assert.assertTrue(format.isFindColumnsFromHeader());
+ }
+
+ @Test
+ public void testFindColumnsFromHeaderWithMissingColumnsReturningItsValue()
+ {
+ final DelimitedInputFormat format = new DelimitedInputFormat(
+ null,
+ null,
+ "delim",
+ null,
+ true,
+ 0
+ );
+ Assert.assertTrue(format.isFindColumnsFromHeader());
+ }
+
+ @Test
+ public void testMissingFindColumnsFromHeaderWithMissingColumnsThrowingError()
+ {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("At least one of [Property{name='hasHeaderRow', value=null}");
+ new DelimitedInputFormat(null, null, "delim", null, null, 0);
+ }
+
+ @Test
+ public void testMissingFindColumnsFromHeaderWithColumnsReturningFalse()
+ {
+ final DelimitedInputFormat format = new DelimitedInputFormat(
+ Collections.singletonList("a"),
+ null,
+ "delim",
+ null,
+ null,
+ 0
+ );
+ Assert.assertFalse(format.isFindColumnsFromHeader());
+ }
+
+ @Test
+ public void testHasHeaderRowWithMissingFindColumnsThrowingError()
+ {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("At most one of [Property{name='hasHeaderRow', value=true}");
+ new DelimitedInputFormat(null, null, "delim", true, false, 0);
+ }
+
+ @Test
+ public void testHasHeaderRowWithMissingColumnsReturningItsValue()
+ {
+ final DelimitedInputFormat format = new DelimitedInputFormat(null, null, "delim", true, null, 0);
+ Assert.assertTrue(format.isFindColumnsFromHeader());
+ }
}
diff --git a/core/src/test/java/org/apache/druid/data/input/impl/DelimitedReaderTest.java b/core/src/test/java/org/apache/druid/data/input/impl/DelimitedReaderTest.java
index 5f3bce4..e590ed5 100644
--- a/core/src/test/java/org/apache/druid/data/input/impl/DelimitedReaderTest.java
+++ b/core/src/test/java/org/apache/druid/data/input/impl/DelimitedReaderTest.java
@@ -20,13 +20,11 @@
package org.apache.druid.data.input.impl;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.InputEntityReader;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.InputRowSchema;
-import org.apache.druid.data.input.MapBasedInputRow;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.parsers.CloseableIterator;
@@ -38,7 +36,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
@@ -201,105 +198,11 @@ public class DelimitedReaderTest
}
@Test
- public void testQuotes() throws IOException
- {
- final ByteEntity source = writeData(
- ImmutableList.of(
- "3\t\"Lets do some \"\"normal\"\" quotes\"\t2018-05-05T10:00:00Z",
- "34\t\"Lets do some \"\"normal\"\", quotes with comma\"\t2018-05-06T10:00:00Z",
- "343\t\"Lets try \\\"\"it\\\"\" with slash quotes\"\t2018-05-07T10:00:00Z",
- "545\t\"Lets try \\\"\"it\\\"\", with slash quotes and comma\"\t2018-05-08T10:00:00Z",
- "65\tHere I write \\n slash n\t2018-05-09T10:00:00Z"
- )
- );
- final List<InputRow> expectedResults = ImmutableList.of(
- new MapBasedInputRow(
- DateTimes.of("2018-05-05T10:00:00Z"),
- ImmutableList.of("Timestamp"),
- ImmutableMap.of(
- "Value",
- "3",
- "Comment",
- "Lets do some \"normal\" quotes",
- "Timestamp",
- "2018-05-05T10:00:00Z"
- )
- ),
- new MapBasedInputRow(
- DateTimes.of("2018-05-06T10:00:00Z"),
- ImmutableList.of("Timestamp"),
- ImmutableMap.of(
- "Value",
- "34",
- "Comment",
- "Lets do some \"normal\", quotes with comma",
- "Timestamp",
- "2018-05-06T10:00:00Z"
- )
- ),
- new MapBasedInputRow(
- DateTimes.of("2018-05-07T10:00:00Z"),
- ImmutableList.of("Timestamp"),
- ImmutableMap.of(
- "Value",
- "343",
- "Comment",
- "Lets try \\\"it\\\" with slash quotes",
- "Timestamp",
- "2018-05-07T10:00:00Z"
- )
- ),
- new MapBasedInputRow(
- DateTimes.of("2018-05-08T10:00:00Z"),
- ImmutableList.of("Timestamp"),
- ImmutableMap.of(
- "Value",
- "545",
- "Comment",
- "Lets try \\\"it\\\", with slash quotes and comma",
- "Timestamp",
- "2018-05-08T10:00:00Z"
- )
- ),
- new MapBasedInputRow(
- DateTimes.of("2018-05-09T10:00:00Z"),
- ImmutableList.of("Timestamp"),
- ImmutableMap.of("Value", "65", "Comment", "Here I write \\n slash n", "Timestamp", "2018-05-09T10:00:00Z")
- )
- );
- final DelimitedInputFormat format = new DelimitedInputFormat(
- ImmutableList.of("Value", "Comment", "Timestamp"),
- null,
- null,
- null,
- false,
- 0
- );
- final InputEntityReader reader = format.createReader(
- new InputRowSchema(
- new TimestampSpec("Timestamp", "auto", null),
- new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("Timestamp"))),
- Collections.emptyList()
- ),
- source,
- null
- );
-
- try (CloseableIterator<InputRow> iterator = reader.read()) {
- final Iterator<InputRow> expectedRowIterator = expectedResults.iterator();
- while (iterator.hasNext()) {
- Assert.assertTrue(expectedRowIterator.hasNext());
- Assert.assertEquals(expectedRowIterator.next(), iterator.next());
- }
- }
- }
-
- @Test
public void testRussianTextMess() throws IOException
{
final ByteEntity source = writeData(
ImmutableList.of(
- "2019-01-01T00:00:10Z\tname_1\t\"Как говорится: \\\"\"всё течет\t всё изменяется\\\"\". Украина как всегда обвиняет Россию в собственных проблемах. #ПровокацияКиева\""
+ "2019-01-01T00:00:10Z\tname_1\tКак говорится: \\\"всё течет всё изменяется\\\". Украина как всегда обвиняет Россию в собственных проблемах. #ПровокацияКиева"
)
);
final DelimitedInputFormat format = new DelimitedInputFormat(
@@ -317,7 +220,7 @@ public class DelimitedReaderTest
Assert.assertEquals(DateTimes.of("2019-01-01T00:00:10Z"), row.getTimestamp());
Assert.assertEquals("name_1", Iterables.getOnlyElement(row.getDimension("name")));
Assert.assertEquals(
- "Как говорится: \\\"всё течет\t всё изменяется\\\". Украина как всегда обвиняет Россию в собственных проблемах. #ПровокацияКиева",
+ "Как говорится: \\\"всё течет всё изменяется\\\". Украина как всегда обвиняет Россию в собственных проблемах. #ПровокацияКиева",
Iterables.getOnlyElement(row.getDimension("Comment"))
);
Assert.assertFalse(iterator.hasNext());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org