You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@johnzon.apache.org by st...@apache.org on 2019/08/20 18:50:55 UTC
[johnzon] branch master updated: JOHNZON-271 fix bugs in our
JsonParser Stream handling
This is an automated email from the ASF dual-hosted git repository.
struberg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/johnzon.git
The following commit(s) were added to refs/heads/master by this push:
new 9871e29 JOHNZON-271 fix bugs in our JsonParser Stream handling
9871e29 is described below
commit 9871e297994a092d8e2492c58686bac6c23b56a4
Author: Mark Struberg <st...@apache.org>
AuthorDate: Tue Aug 20 20:50:05 2019 +0200
JOHNZON-271 fix bugs in our JsonParser Stream handling
problems uncovered by running the JSON-P TCK which we now pass.
---
johnzon-core/pom.xml | 23 +++++
.../org/apache/johnzon/core/JohnzonJsonParser.java | 2 +-
.../apache/johnzon/core/JohnzonJsonParserImpl.java | 62 +++++++++----
.../apache/johnzon/core/JsonInMemoryParser.java | 15 ++-
.../apache/johnzon/core/JsonStreamParserImpl.java | 46 +++-------
.../johnzon/core/JsonParserStreamingTest.java | 102 +++++++++++++++++++++
.../org/apache/johnzon/core/ManualTckTest.java | 60 ++++++++++++
7 files changed, 257 insertions(+), 53 deletions(-)
diff --git a/johnzon-core/pom.xml b/johnzon-core/pom.xml
index dbe6d03..678d71c 100644
--- a/johnzon-core/pom.xml
+++ b/johnzon-core/pom.xml
@@ -47,4 +47,27 @@
</plugin>
</plugins>
</build>
+
+ <profiles>
+ <profile>
+ <id>jakartaee-tck</id>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.jakartaee.tck.local</groupId>
+ <artifactId>tsharness</artifactId>
+ <version>8.0-SNAPSHOT</version>
+ <scope>system</scope>
+ <systemPath>/opt/eclipse/javaeetck/lib/tsharness.jar</systemPath>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jakartaee.tck.local</groupId>
+ <artifactId>sonparsertests_appclient_vehicle_client</artifactId>
+ <version>8.0-SNAPSHOT</version>
+ <scope>system</scope>
+ <systemPath>/opt/eclipse/javaeetck/dist/com/sun/ts/tests/jsonp/api/jsonparsertests/jsonparsertests_appclient_vehicle_client.jar</systemPath>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
diff --git a/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParser.java b/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParser.java
index cb0a40f..c26ca10 100644
--- a/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParser.java
+++ b/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParser.java
@@ -43,7 +43,7 @@ public interface JohnzonJsonParser extends JsonParser {
}
- public static class JohnzonJsonParserWrapper implements JohnzonJsonParser {
+ class JohnzonJsonParserWrapper implements JohnzonJsonParser {
private final JsonParser jsonParser;
public JohnzonJsonParserWrapper(JsonParser jsonParser) {
diff --git a/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParserImpl.java b/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParserImpl.java
index eaa6ed1..cf64a36 100644
--- a/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParserImpl.java
+++ b/johnzon-core/src/main/java/org/apache/johnzon/core/JohnzonJsonParserImpl.java
@@ -17,6 +17,7 @@
package org.apache.johnzon.core;
+import java.util.Collections;
import java.util.Map;
import java.util.stream.Stream;
@@ -34,8 +35,23 @@ public abstract class JohnzonJsonParserImpl implements JohnzonJsonParser {
* @return {@code true} if we are currently inside an array
*/
protected abstract boolean isInArray();
+ /**
+ * @return {@code true} if we are currently inside an object
+ */
+ protected abstract boolean isInObject();
+
protected abstract BufferStrategy.BufferProvider<char[]> getCharArrayProvider();
+ private boolean manualNext = false;
+
+ @Override
+ public Event next() {
+ manualNext = true;
+ return internalNext();
+ }
+
+ protected abstract Event internalNext();
+
@Override
public JsonObject getObject() {
Event current = current();
@@ -88,15 +104,17 @@ public abstract class JohnzonJsonParserImpl implements JohnzonJsonParser {
@Override
public void skipObject() {
- int level = 1;
- do {
- Event event = next();
- if (event == Event.START_OBJECT) {
- level++;
- } else if (event == Event.END_OBJECT) {
- level --;
- }
- } while (level > 0 && hasNext());
+ if (isInObject()) {
+ int level = 1;
+ do {
+ Event event = internalNext();
+ if (event == Event.START_OBJECT) {
+ level++;
+ } else if (event == Event.END_OBJECT) {
+ level--;
+ }
+ } while (level > 0 && hasNext());
+ }
}
@Override
@@ -104,7 +122,7 @@ public abstract class JohnzonJsonParserImpl implements JohnzonJsonParser {
if (isInArray()) {
int level = 1;
do {
- Event event = next();
+ Event event = internalNext();
if (event == Event.START_ARRAY) {
level++;
} else if (event == Event.END_ARRAY) {
@@ -132,16 +150,22 @@ public abstract class JohnzonJsonParserImpl implements JohnzonJsonParser {
@Override
public Stream<JsonValue> getValueStream() {
- //X TODO this implementation is very simplistic
- //X I find it unintuitive what the spec intends here
- //X we probably need to improve this
- Event current = current();
- if (current == Event.START_ARRAY) {
- return getArrayStream();
+ if (manualNext) {
+ throw new IllegalStateException("JsonStream already got propagated manually");
}
- if (current == Event.START_OBJECT) {
- return getObject().values().stream();
+
+ Event event = internalNext();
+ switch (event) {
+ case START_ARRAY:
+ case START_OBJECT:
+ case VALUE_STRING:
+ case VALUE_NUMBER:
+ case VALUE_TRUE:
+ case VALUE_FALSE:
+ case VALUE_NULL:
+ return Collections.singletonList(getValue()).stream();
+ default:
+ throw new IllegalStateException(event + " doesn't support getValueStream");
}
- throw new IllegalStateException(current + " doesn't support getValueStream");
}
}
diff --git a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonInMemoryParser.java b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonInMemoryParser.java
index fcea38a..740a100 100644
--- a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonInMemoryParser.java
+++ b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonInMemoryParser.java
@@ -39,6 +39,7 @@ class JsonInMemoryParser extends JohnzonJsonParserImpl {
private Event currentEvent;
private JsonValue currentValue;
private int arrayDepth = 0;
+ private int objectDepth = 0;
private class ArrayIterator implements Iterator<Event> {
@@ -175,7 +176,7 @@ class JsonInMemoryParser extends JohnzonJsonParserImpl {
@Override
public Event current() {
if (currentEvent == null && hasNext()) {
- next();
+ internalNext();
}
return currentEvent;
}
@@ -186,6 +187,11 @@ class JsonInMemoryParser extends JohnzonJsonParserImpl {
}
@Override
+ protected boolean isInObject() {
+ return objectDepth > 0;
+ }
+
+ @Override
protected BufferStrategy.BufferProvider<char[]> getCharArrayProvider() {
return bufferProvider;
}
@@ -226,7 +232,7 @@ class JsonInMemoryParser extends JohnzonJsonParserImpl {
}
@Override
- public Event next() {
+ protected Event internalNext() {
if (!hasNext()) {
throw new NoSuchElementException();
@@ -239,6 +245,11 @@ class JsonInMemoryParser extends JohnzonJsonParserImpl {
} else if (currentEvent == Event.END_ARRAY) {
arrayDepth--;
}
+ if (currentEvent == Event.START_OBJECT) {
+ objectDepth++;
+ } else if (currentEvent == Event.END_OBJECT) {
+ objectDepth--;
+ }
return currentEvent;
}
diff --git a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java
index c6f9da8..fd05752 100644
--- a/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java
+++ b/johnzon-core/src/main/java/org/apache/johnzon/core/JsonStreamParserImpl.java
@@ -94,6 +94,7 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
private StructureElement currentStructureElement = null;
private int arrayDepth = 0;
+ private int objectDepth = 0;
private boolean closed;
@@ -351,7 +352,7 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
@Override
public Event current() {
if (previousEvent < 0 && hasNext()) {
- next();
+ internalNext();
}
return previousEvent >= 0 && previousEvent < Event.values().length
? Event.values()[previousEvent]
@@ -359,7 +360,7 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
}
@Override
- public final Event next() {
+ protected final Event internalNext() {
//main entry, make decision how to handle the current character in the stream
if (!hasNext()) {
@@ -386,7 +387,7 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
}
previousEvent = COMMA_EVENT;
- return next();
+ return internalNext();
}
@@ -397,7 +398,7 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
}
previousEvent = KEY_SEPARATOR_EVENT;
- return next();
+ return internalNext();
}
@@ -488,6 +489,8 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
currentStructureElement = localStructureElement;
}
+ objectDepth++;
+
return EVT_MAP[previousEvent = START_OBJECT];
}
@@ -507,6 +510,8 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
//pop from stack
currentStructureElement = currentStructureElement.previous;
+ objectDepth--;
+
return EVT_MAP[previousEvent = END_OBJECT];
}
@@ -558,6 +563,12 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
return arrayDepth > 0;
}
+
+ @Override
+ protected boolean isInObject() {
+ return objectDepth > 0;
+ }
+
@Override
protected BufferStrategy.BufferProvider<char[]> getCharArrayProvider() {
return bufferProvider;
@@ -651,33 +662,6 @@ public class JsonStreamParserImpl extends JohnzonJsonParserImpl implements JsonC
}
- //maybe we want to check invalid utf encoding
- //not clear yet if the InputStreamReader is doing that
-
- /*
- private char checkSurrogates(char n, char highSurrogate) {
- //check for invalid surrogates
- //high followed by low
- if (Character.isHighSurrogate(n)) {
-
- if (highSurrogate != 0) {
- throw uexc("Unexpected high surrogate");
- }
- return n;
- } else if (Character.isLowSurrogate(n)) {
-
- if (highSurrogate == 0) {
- throw uexc("Unexpected low surrogate");
- } else if (!Character.isSurrogatePair(highSurrogate, n)) {
- throw uexc("Invalid surrogate pair");
- }
- return 0;
- } else if (highSurrogate != 0 && !Character.isLowSurrogate(n)) {
- throw uexc("Expected low surrogate");
- }
-
- return highSurrogate;
- }*/
//read the next four chars, check them and treat them as an single unicode char
private char parseUnicodeHexChars() {
diff --git a/johnzon-core/src/test/java/org/apache/johnzon/core/JsonParserStreamingTest.java b/johnzon-core/src/test/java/org/apache/johnzon/core/JsonParserStreamingTest.java
new file mode 100644
index 0000000..2cc5d21
--- /dev/null
+++ b/johnzon-core/src/test/java/org/apache/johnzon/core/JsonParserStreamingTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.johnzon.core;
+
+import java.io.StringReader;
+import java.util.stream.Collectors;
+
+import javax.json.Json;
+import javax.json.stream.JsonParser;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class JsonParserStreamingTest {
+
+ @Test
+ public void testArrayStream() {
+ {
+ // int
+ String json = "[1,2,3,4]";
+ String sum = parserAndConcat(json);
+ assertEquals(json, sum);
+ }
+ {
+ // STring
+ String json = "[\"1\",\"2\",\"3\",\"4\"]";
+ String sum = parserAndConcat(json);
+ assertEquals(json, sum);
+ }
+ }
+
+ @Test
+ public void testValueStreamForArrays() {
+ {
+ // int
+ String json = "[1,2,3,4]";
+ String sum = parserAndConcat(json);
+ assertEquals(json, sum);
+ }
+ {
+ // String
+ String json = "[\"1\",\"2\",\"3\",\"4\"]";
+ String sum = parserAndConcat(json);
+ assertEquals(json, sum);
+ }
+
+ {
+ // single String
+ String json = "[\"In a galaxy far far away\"]";
+ String sum = parserAndConcat(json);
+ assertEquals(json, sum);
+ }
+ }
+
+ @Test
+ public void testValueStream() {
+ {
+ String json = "\"a string value\"";
+ String sum = parserAndConcat(json);
+ assertEquals(json, sum);
+ }
+
+ {
+ String sum = parserAndConcat("234");
+ assertEquals("234", sum);
+ }
+ }
+
+ @Test
+ public void testValueStreamForObjects() {
+ String json = "{\"address\":\"somewhere else\"}";
+ String sum = parserAndConcat(json);
+ assertEquals(json, sum);
+ }
+
+ private String parserAndConcat(String json) {
+ StringReader sr = new StringReader(json);
+ JsonParser jsonParser = Json.createParser(sr);
+
+ String sum = jsonParser.getValueStream()
+ .map(v -> v.toString())
+ .collect(Collectors.joining(","));
+ return sum;
+ }
+}
diff --git a/johnzon-core/src/test/java/org/apache/johnzon/core/ManualTckTest.java b/johnzon-core/src/test/java/org/apache/johnzon/core/ManualTckTest.java
new file mode 100644
index 0000000..afb152d
--- /dev/null
+++ b/johnzon-core/src/test/java/org/apache/johnzon/core/ManualTckTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.johnzon.core;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+
+public class ManualTckTest {
+
+
+ @Test
+ public void jsonParser11Test() throws Exception {
+ Class parserClass = loadClass("com.sun.ts.tests.jsonp.api.jsonparsertests.Parser");
+ if (parserClass == null) {
+ // no TCK available, so we skip the test
+ return;
+ }
+
+ Constructor constructor = parserClass.getDeclaredConstructor();
+ constructor.setAccessible(true);
+ Object parserTest = constructor.newInstance();
+ assertNotNull(parserTest);
+
+ Method testMethod = parserClass.getDeclaredMethod("test");
+ testMethod.setAccessible(true);
+ final Object result = testMethod.invoke(parserTest);
+
+ Class testResultClass = loadClass("com.sun.ts.tests.jsonp.api.common.TestResult");
+ Method evalMethod = testResultClass.getDeclaredMethod("eval");
+ evalMethod.invoke(result);
+ }
+
+ private Class loadClass(String className) {
+ try {
+ return Class.forName(className);
+ } catch (ClassNotFoundException cnfe) {
+ return null;
+ }
+ }
+}