You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/12/01 21:17:53 UTC
[arrow] branch master updated: ARROW-1817: [Java] Configure
JsonReader to read floating point NaN values
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new ad9105e ARROW-1817: [Java] Configure JsonReader to read floating point NaN values
ad9105e is described below
commit ad9105e639b115e2868483b534fb3ac7a8921de3
Author: Bryan Cutler <cu...@gmail.com>
AuthorDate: Fri Dec 1 16:17:49 2017 -0500
ARROW-1817: [Java] Configure JsonReader to read floating point NaN values
The last upgrade of the Jackson JSON library changed behavior to no longer allow reading of "NaN" values by default. This change configures the JSON generator and parser to allow for NaN values (unquoted) alongside standard floating point numbers. A test was added for JSON writing/reading and modified the test for Arrow file and stream .
Author: Bryan Cutler <cu...@gmail.com>
Closes #1375 from BryanCutler/java-JsonReader-all_non_numeric-ARROW-1817 and squashes the following commits:
4c4682a9 [Bryan Cutler] configure JsonWriter to write NaN not as strings, add test for read and write of float with NaN
1fa24f45 [Bryan Cutler] added conf for JacksonParser to allow NaN tokens
---
.../apache/arrow/vector/ipc/JsonFileReader.java | 2 ++
.../apache/arrow/vector/ipc/JsonFileWriter.java | 2 ++
.../org/apache/arrow/vector/ipc/BaseFileTest.java | 33 +++++++++++++---------
.../org/apache/arrow/vector/ipc/TestJSONFile.java | 27 ++++++++++++++++++
4 files changed, 50 insertions(+), 14 deletions(-)
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index 04d5033..f274b74 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -69,6 +69,8 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider {
this.allocator = allocator;
MappingJsonFactory jsonFactory = new MappingJsonFactory();
this.parser = jsonFactory.createParser(inputFile);
+ // Allow reading NaN for floating point values
+ this.parser.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, true);
}
@Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index 067fb25..1c9e1d3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -84,6 +84,8 @@ public class JsonFileWriter implements AutoCloseable {
prettyPrinter.indentArraysWith(NopIndenter.instance);
this.generator.setPrettyPrinter(prettyPrinter);
}
+ // Allow writing of floating point NaN values not as strings
+ this.generator.configure(JsonGenerator.Feature.QUOTE_NON_NUMERIC_NUMBERS, false);
}
public void start(Schema schema, DictionaryProvider provider) throws IOException {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 3514aca..d26385d 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -47,6 +47,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.arrow.vector.complex.writer.BigIntWriter;
import org.apache.arrow.vector.complex.writer.DateMilliWriter;
+import org.apache.arrow.vector.complex.writer.Float4Writer;
import org.apache.arrow.vector.complex.writer.IntWriter;
import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
import org.apache.arrow.vector.complex.writer.TimeStampMilliTZWriter;
@@ -95,10 +96,28 @@ public class BaseFileTest {
DateTimeZone.setDefault(defaultTimezone);
}
+ protected void writeData(int count, MapVector parent) {
+ ComplexWriter writer = new ComplexWriterImpl("root", parent);
+ MapWriter rootWriter = writer.rootAsMap();
+ IntWriter intWriter = rootWriter.integer("int");
+ BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
+ Float4Writer float4Writer = rootWriter.float4("float");
+ for (int i = 0; i < count; i++) {
+ intWriter.setPosition(i);
+ intWriter.writeInt(i);
+ bigIntWriter.setPosition(i);
+ bigIntWriter.writeBigInt(i);
+ float4Writer.setPosition(i);
+ float4Writer.writeFloat4(i == 0 ? Float.NaN : i);
+ }
+ writer.setValueCount(count);
+ }
+
protected void validateContent(int count, VectorSchemaRoot root) {
for (int i = 0; i < count; i++) {
Assert.assertEquals(i, root.getVector("int").getObject(i));
Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i));
+ Assert.assertEquals(i == 0 ? Float.NaN : i, root.getVector("float").getObject(i));
}
}
@@ -454,20 +473,6 @@ public class BaseFileTest {
}
}
- protected void writeData(int count, MapVector parent) {
- ComplexWriter writer = new ComplexWriterImpl("root", parent);
- MapWriter rootWriter = writer.rootAsMap();
- IntWriter intWriter = rootWriter.integer("int");
- BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt");
- for (int i = 0; i < count; i++) {
- intWriter.setPosition(i);
- intWriter.writeInt(i);
- bigIntWriter.setPosition(i);
- bigIntWriter.writeBigInt(i);
- }
- writer.setValueCount(count);
- }
-
public void validateUnionData(int count, VectorSchemaRoot root) {
FieldReader unionReader = root.getVector("union").getReader();
for (int i = 0; i < count; i++) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
index 6257170..4f9093b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java
@@ -39,6 +39,33 @@ public class TestJSONFile extends BaseFileTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TestJSONFile.class);
@Test
+ public void testWriteRead() throws IOException {
+ File file = new File("target/mytest.json");
+ int count = COUNT;
+
+ // write
+ try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+ MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
+ writeData(count, parent);
+ writeJSON(file, new VectorSchemaRoot(parent.getChild("root")), null);
+ }
+
+ // read
+ try (
+ BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+ JsonFileReader reader = new JsonFileReader(file, readerAllocator)
+ ) {
+ Schema schema = reader.start();
+ LOGGER.debug("reading schema: " + schema);
+
+ // initialize vectors
+ try (VectorSchemaRoot root = reader.read();) {
+ validateContent(count, root);
+ }
+ }
+ }
+
+ @Test
public void testWriteReadComplexJSON() throws IOException {
File file = new File("target/mytest_complex.json");
int count = COUNT;
--
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <co...@arrow.apache.org>'].