You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/05/16 02:07:25 UTC
[lucene-solr] branch branch_8x updated: SOLR-13437: fork noggit
code into Solr (#666)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 0f67df4 SOLR-13437: fork noggit code into Solr (#666)
0f67df4 is described below
commit 0f67df40b6916aab076ca1b2312e85547ac5693d
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Thu May 16 11:10:27 2019 +1000
SOLR-13437: fork noggit code into Solr (#666)
* SOLR-13437: fork noggit code into Solr
---
lucene/ivy-versions.properties | 1 -
solr/CHANGES.txt | 2 +
solr/NOTICE.txt | 15 +-
.../solr/schema/TestSchemalessBufferedUpdates.java | 2 +-
solr/solrj/ivy.xml | 1 -
.../java/org/apache/solr/common/cloud/Replica.java | 4 +-
.../org/apache/solr/common/cloud/RoutingRule.java | 4 +-
.../org/apache/solr/common/util/ByteUtils.java | 1 +
solr/solrj/src/java/org/noggit/CharArr.java | 394 ++++++
solr/solrj/src/java/org/noggit/JSONParser.java | 1297 ++++++++++++++++++++
solr/solrj/src/java/org/noggit/JSONUtil.java | 203 +++
solr/solrj/src/java/org/noggit/JSONWriter.java | 358 ++++++
solr/solrj/src/java/org/noggit/ObjectBuilder.java | 168 +++
solr/solrj/src/test/org/noggit/TestJSONParser.java | 690 +++++++++++
solr/solrj/src/test/org/noggit/TestJSONWriter.java | 94 ++
.../src/test/org/noggit/TestObjectBuilder.java | 99 ++
16 files changed, 3325 insertions(+), 8 deletions(-)
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 98c94d7..7767b7c 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -274,7 +274,6 @@ org.gagravarr.vorbis.java.version = 0.8
/org.mockito/mockito-core = 2.23.4
-/org.noggit/noggit = 0.8
/org.objenesis/objenesis = 2.6
org.ow2.asm.version = 5.1
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ebc25ed..e2691a6 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -51,6 +51,8 @@ Other Changes
* SOLR-13453: Adjust auth metrics asserts in tests caused by SOLR-13449 (janhoy)
+* SOLR-13437: noggit json parser is forked into solrj (noble)
+
================== 8.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
diff --git a/solr/NOTICE.txt b/solr/NOTICE.txt
index daeddbb..760d650 100644
--- a/solr/NOTICE.txt
+++ b/solr/NOTICE.txt
@@ -590,4 +590,17 @@ The Base64 implementation in this software was derived from the
Apache Commons Codec project. http://commons.apache.org/proper/commons-codec/
JSON processing in this software was derived from the JSON.simple toolkit.
-https://code.google.com/p/json-simple/
\ No newline at end of file
+https://code.google.com/p/json-simple/
+
+=========================================================================
+== noggit notice ==
+=========================================================================
+
+noggit
+
+Copyright 2006- Yonik Seeley
+
+Noggit is a fast streaming JSON parser for java. The code is included
+into Solr codebase.
+
+https://github.com/yonik/noggit
diff --git a/solr/core/src/test/org/apache/solr/schema/TestSchemalessBufferedUpdates.java b/solr/core/src/test/org/apache/solr/schema/TestSchemalessBufferedUpdates.java
index eb1031c..96ee4c6 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestSchemalessBufferedUpdates.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestSchemalessBufferedUpdates.java
@@ -145,7 +145,7 @@ public class TestSchemalessBufferedUpdates extends SolrTestCaseJ4 {
UpdateRequestProcessor processor = chainUpToDUP.createProcessor(req, rsp);
processor.processAdd(cmd);
if (cmd.solrDoc.get("f_dt").getValue() instanceof Date) {
- // Non-JSON types (Date in this case) aren't handled properly in noggit-0.6. Although this is fixed in
+ // Non-JSON types (Date in this case) aren't handled properly in noggit-0.6. Although this is fixed in
// https://github.com/yonik/noggit/commit/ec3e732af7c9425e8f40297463cbe294154682b1 to call obj.toString(),
// Date::toString produces a Date representation that Solr doesn't like, so we convert using Instant::toString
cmd.solrDoc.get("f_dt").setValue(((Date) cmd.solrDoc.get("f_dt").getValue()).toInstant().toString());
diff --git a/solr/solrj/ivy.xml b/solr/solrj/ivy.xml
index 85db8ce..c158510 100644
--- a/solr/solrj/ivy.xml
+++ b/solr/solrj/ivy.xml
@@ -36,7 +36,6 @@
<dependency org="org.apache.commons" name="commons-math3" rev="${/org.apache.commons/commons-math3}" conf="compile"/>
<dependency org="org.codehaus.woodstox" name="woodstox-core-asl" rev="${/org.codehaus.woodstox/woodstox-core-asl}" conf="compile"/>
<dependency org="org.codehaus.woodstox" name="stax2-api" rev="${/org.codehaus.woodstox/stax2-api}" conf="compile"/>
- <dependency org="org.noggit" name="noggit" rev="${/org.noggit/noggit}" conf="compile"/>
<dependency org="org.slf4j" name="slf4j-api" rev="${/org.slf4j/slf4j-api}" conf="compile"/>
<dependency org="org.slf4j" name="jcl-over-slf4j" rev="${/org.slf4j/jcl-over-slf4j}" conf="compile"/>
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java b/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
index d73282b..e3b6d5d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
@@ -20,7 +20,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import org.noggit.JSONUtil;
+import org.apache.solr.common.util.Utils;
public class Replica extends ZkNodeProps {
@@ -183,6 +183,6 @@ public class Replica extends ZkNodeProps {
@Override
public String toString() {
- return name + ':' + JSONUtil.toJSON(propMap, -1); // small enough, keep it on one line (i.e. no indent)
+ return name + ':' + Utils.toJSONString(propMap); // small enough, keep it on one line (i.e. no indent)
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/RoutingRule.java b/solr/solrj/src/java/org/apache/solr/common/cloud/RoutingRule.java
index 503c39e..5e8e050 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/RoutingRule.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/RoutingRule.java
@@ -21,7 +21,7 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.common.util.SuppressForbidden;
-import org.noggit.JSONUtil;
+import org.apache.solr.common.util.Utils;
/**
* Used for routing docs with particular keys into another collection
@@ -72,6 +72,6 @@ public class RoutingRule extends ZkNodeProps {
@Override
public String toString() {
- return JSONUtil.toJSON(propMap, -1);
+ return Utils.toJSONString(propMap);
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java b/solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java
index ec4acdb..be64e6c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ByteUtils.java
@@ -21,6 +21,7 @@ import java.io.OutputStream;
import org.noggit.CharArr;
+
public class ByteUtils {
/** Maximum number of UTF8 bytes per UTF16 character. */
diff --git a/solr/solrj/src/java/org/noggit/CharArr.java b/solr/solrj/src/java/org/noggit/CharArr.java
new file mode 100644
index 0000000..9ecc8e6
--- /dev/null
+++ b/solr/solrj/src/java/org/noggit/CharArr.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.CharBuffer;
+
+public class CharArr implements CharSequence, Appendable {
+ protected char[] buf;
+ protected int start;
+ protected int end;
+
+ public CharArr() {
+ this(32);
+ }
+
+ public CharArr(int size) {
+ buf = new char[size];
+ }
+
+ public CharArr(char[] arr, int start, int end) {
+ set(arr, start, end);
+ }
+
+ public void setStart(int start) {
+ this.start = start;
+ }
+
+ public void setEnd(int end) {
+ this.end = end;
+ }
+
+ public void set(char[] arr, int start, int end) {
+ this.buf = arr;
+ this.start = start;
+ this.end = end;
+ }
+
+ public char[] getArray() {
+ return buf;
+ }
+
+ public int getStart() {
+ return start;
+ }
+
+ public int getEnd() {
+ return end;
+ }
+
+ public int size() {
+ return end - start;
+ }
+
+ @Override
+ public int length() {
+ return size();
+ }
+
+ /**
+ * The capacity of the buffer when empty (getArray().size())
+ */
+ public int capacity() {
+ return buf.length;
+ }
+
+
+ @Override
+ public char charAt(int index) {
+ return buf[start + index];
+ }
+
+ @Override
+ public CharArr subSequence(int start, int end) {
+ return new CharArr(buf, this.start + start, this.start + end);
+ }
+
+ public int read() throws IOException {
+ if (start >= end) return -1;
+ return buf[start++];
+ }
+
+ public int read(char cbuf[], int off, int len) {
+ //TODO
+ return 0;
+ }
+
+ public void unsafeWrite(char b) {
+ buf[end++] = b;
+ }
+
+ public void unsafeWrite(int b) {
+ unsafeWrite((char) b);
+ }
+
+ public void unsafeWrite(char b[], int off, int len) {
+ System.arraycopy(b, off, buf, end, len);
+ end += len;
+ }
+
+ protected void resize(int len) {
+ char newbuf[] = new char[Math.max(buf.length << 1, len)];
+ System.arraycopy(buf, start, newbuf, 0, size());
+ buf = newbuf;
+ }
+
+ public void reserve(int num) {
+ if (end + num > buf.length) resize(end + num);
+ }
+
+ public void write(char b) {
+ if (end >= buf.length) {
+ resize(end + 1);
+ }
+ unsafeWrite(b);
+ }
+
+ public final void write(int b) {
+ write((char) b);
+ }
+
+ public final void write(char[] b) {
+ write(b, 0, b.length);
+ }
+
+ public void write(char b[], int off, int len) {
+ reserve(len);
+ unsafeWrite(b, off, len);
+ }
+
+ public final void write(CharArr arr) {
+ write(arr.buf, arr.start, arr.end - arr.start);
+ }
+
+ public final void write(String s) {
+ write(s, 0, s.length());
+ }
+
+ public void write(String s, int stringOffset, int len) {
+ reserve(len);
+ s.getChars(stringOffset, len, buf, end);
+ end += len;
+ }
+
+ public void flush() {
+ }
+
+ public final void reset() {
+ start = end = 0;
+ }
+
+ public void close() {
+ }
+
+ public char[] toCharArray() {
+ char newbuf[] = new char[size()];
+ System.arraycopy(buf, start, newbuf, 0, size());
+ return newbuf;
+ }
+
+
+ @Override
+ public String toString() {
+ return new String(buf, start, size());
+ }
+
+ public int read(CharBuffer cb) throws IOException {
+
+ /***
+ int sz = size();
+ if (sz<=0) return -1;
+ if (sz>0) cb.put(buf, start, sz);
+ return -1;
+ ***/
+
+ int sz = size();
+ if (sz > 0) cb.put(buf, start, sz);
+ start = end;
+ while (true) {
+ fill();
+ int s = size();
+ if (s == 0) return sz == 0 ? -1 : sz;
+ sz += s;
+ cb.put(buf, start, s);
+ }
+ }
+
+
+ public int fill() throws IOException {
+ return 0; // or -1?
+ }
+
+ //////////////// Appendable methods /////////////
+ @Override
+ public final Appendable append(CharSequence csq) throws IOException {
+ return append(csq, 0, csq.length());
+ }
+
+ @Override
+ public Appendable append(CharSequence csq, int start, int end) throws IOException {
+ write(csq.subSequence(start, end).toString());
+ return null;
+ }
+
+ @Override
+ public final Appendable append(char c) throws IOException {
+ write(c);
+ return this;
+ }
+}
+
+
+class NullCharArr extends CharArr {
+ public NullCharArr() {
+ super(new char[1], 0, 0);
+ }
+
+ @Override
+ public void unsafeWrite(char b) {
+ }
+
+ @Override
+ public void unsafeWrite(char b[], int off, int len) {
+ }
+
+ @Override
+ public void unsafeWrite(int b) {
+ }
+
+ @Override
+ public void write(char b) {
+ }
+
+ @Override
+ public void write(char b[], int off, int len) {
+ }
+
+ @Override
+ public void reserve(int num) {
+ }
+
+ @Override
+ protected void resize(int len) {
+ }
+
+ @Override
+ public Appendable append(CharSequence csq, int start, int end) throws IOException {
+ return this;
+ }
+
+ @Override
+ public char charAt(int index) {
+ return 0;
+ }
+
+ @Override
+ public void write(String s, int stringOffset, int len) {
+ }
+}
+
+
+// IDEA: a subclass that refills the array from a reader?
+class CharArrReader extends CharArr {
+ protected final Reader in;
+
+ public CharArrReader(Reader in, int size) {
+ super(size);
+ this.in = in;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (start >= end) fill();
+ return start >= end ? -1 : buf[start++];
+ }
+
+ @Override
+ public int read(CharBuffer cb) throws IOException {
+ // empty the buffer and then read direct
+ int sz = size();
+ if (sz > 0) cb.put(buf, start, end);
+ int sz2 = in.read(cb);
+ if (sz2 >= 0) return sz + sz2;
+ return sz > 0 ? sz : -1;
+ }
+
+ @Override
+ public int fill() throws IOException {
+ if (start >= end) {
+ reset();
+ } else if (start > 0) {
+ System.arraycopy(buf, start, buf, 0, size());
+ end = size();
+ start = 0;
+ }
+ /***
+ // fill fully or not???
+ do {
+ int sz = in.read(buf,end,buf.length-end);
+ if (sz==-1) return;
+ end+=sz;
+ } while (end < buf.length);
+ ***/
+
+ int sz = in.read(buf, end, buf.length - end);
+ if (sz > 0) end += sz;
+ return sz;
+ }
+
+}
+
+
+class CharArrWriter extends CharArr {
+ protected Writer sink;
+
+ @Override
+ public void flush() {
+ try {
+ sink.write(buf, start, end - start);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ start = end = 0;
+ }
+
+ @Override
+ public void write(char b) {
+ if (end >= buf.length) {
+ flush();
+ }
+ unsafeWrite(b);
+ }
+
+ @Override
+ public void write(char b[], int off, int len) {
+ int space = buf.length - end;
+ if (len < space) {
+ unsafeWrite(b, off, len);
+ } else if (len < buf.length) {
+ unsafeWrite(b, off, space);
+ flush();
+ unsafeWrite(b, off + space, len - space);
+ } else {
+ flush();
+ try {
+ sink.write(b, off, len);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void write(String s, int stringOffset, int len) {
+ int space = buf.length - end;
+ if (len < space) {
+ s.getChars(stringOffset, stringOffset + len, buf, end);
+ end += len;
+ } else if (len < buf.length) {
+ // if the data to write is small enough, buffer it.
+ s.getChars(stringOffset, stringOffset + space, buf, end);
+ flush();
+ s.getChars(stringOffset + space, stringOffset + len, buf, 0);
+ end = len - space;
+ } else {
+ flush();
+ // don't buffer, just write to sink
+ try {
+ sink.write(s, stringOffset, len);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ }
+}
diff --git a/solr/solrj/src/java/org/noggit/JSONParser.java b/solr/solrj/src/java/org/noggit/JSONParser.java
new file mode 100644
index 0000000..8b1ac01
--- /dev/null
+++ b/solr/solrj/src/java/org/noggit/JSONParser.java
@@ -0,0 +1,1297 @@
+/*
+ * Copyright 2006- Yonik Seeley
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.io.IOException;
+import java.io.Reader;
+
+
+public class JSONParser {
+
+ /**
+ * Event indicating a JSON string value, including member names of objects
+ */
+ public static final int STRING = 1;
+ /**
+ * Event indicating a JSON number value which fits into a signed 64 bit integer
+ */
+ public static final int LONG = 2;
+ /**
+ * Event indicating a JSON number value which has a fractional part or an exponent
+ * and with string length <= 23 chars not including sign. This covers
+ * all representations of normal values for Double.toString().
+ */
+ public static final int NUMBER = 3;
+ /**
+ * Event indicating a JSON number value that was not produced by toString of any
+ * Java primitive numerics such as Double or Long. It is either
+ * an integer outside the range of a 64 bit signed integer, or a floating
+ * point value with a string representation of more than 23 chars.
+ */
+ public static final int BIGNUMBER = 4;
+ /**
+ * Event indicating a JSON boolean
+ */
+ public static final int BOOLEAN = 5;
+ /**
+ * Event indicating a JSON null
+ */
+ public static final int NULL = 6;
+ /**
+ * Event indicating the start of a JSON object
+ */
+ public static final int OBJECT_START = 7;
+ /**
+ * Event indicating the end of a JSON object
+ */
+ public static final int OBJECT_END = 8;
+ /**
+ * Event indicating the start of a JSON array
+ */
+ public static final int ARRAY_START = 9;
+ /**
+ * Event indicating the end of a JSON array
+ */
+ public static final int ARRAY_END = 10;
+ /**
+ * Event indicating the end of input has been reached
+ */
+ public static final int EOF = 11;
+
+
+ /**
+ * Flags to control parsing behavior
+ */
+ public static final int ALLOW_COMMENTS = 1 << 0;
+ public static final int ALLOW_SINGLE_QUOTES = 1 << 1;
+ public static final int ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER = 1 << 2;
+ public static final int ALLOW_UNQUOTED_KEYS = 1 << 3;
+ public static final int ALLOW_UNQUOTED_STRING_VALUES = 1 << 4;
+ /**
+ * ALLOW_EXTRA_COMMAS causes any number of extra commas in arrays and objects to be ignored
+ * Note that a trailing comma in [] would be [,] (hence calling the feature "trailing" commas
+ * is either limiting or misleading. Since trailing commas is fundamentally incompatible with any future
+ * "fill-in-missing-values-with-null", it was decided to extend this feature to handle any
+ * number of extra commas.
+ */
+ public static final int ALLOW_EXTRA_COMMAS = 1 << 5;
+ public static final int ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT = 1 << 6;
+ public static final int OPTIONAL_OUTER_BRACES = 1 << 7;
+
+ public static final int FLAGS_STRICT = 0;
+ public static final int FLAGS_DEFAULT = ALLOW_COMMENTS | ALLOW_SINGLE_QUOTES | ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER | ALLOW_UNQUOTED_KEYS | ALLOW_UNQUOTED_STRING_VALUES | ALLOW_EXTRA_COMMAS;
+
+ public static class ParseException extends RuntimeException {
+ public ParseException(String msg) {
+ super(msg);
+ }
+ }
+
+ public static String getEventString(int e) {
+ switch (e) {
+ case STRING:
+ return "STRING";
+ case LONG:
+ return "LONG";
+ case NUMBER:
+ return "NUMBER";
+ case BIGNUMBER:
+ return "BIGNUMBER";
+ case BOOLEAN:
+ return "BOOLEAN";
+ case NULL:
+ return "NULL";
+ case OBJECT_START:
+ return "OBJECT_START";
+ case OBJECT_END:
+ return "OBJECT_END";
+ case ARRAY_START:
+ return "ARRAY_START";
+ case ARRAY_END:
+ return "ARRAY_END";
+ case EOF:
+ return "EOF";
+ }
+ return "Unknown: " + e;
+ }
+
+ private static final CharArr devNull = new NullCharArr();
+
+ protected int flags = FLAGS_DEFAULT;
+
+ protected final char[] buf; // input buffer with JSON text in it
+ protected int start; // current position in the buffer
+ protected int end; // end position in the buffer (one past last valid index)
+ protected final Reader in; // optional reader to obtain data from
+ protected boolean eof = false; // true if the end of the stream was reached.
+ protected long gpos; // global position = gpos + start
+
+ protected int event; // last event read
+
+ protected int stringTerm; // The terminator for the last string we read: single quote, double quote, or 0 for unterminated.
+
+ protected boolean missingOpeningBrace = false;
+
+ public JSONParser(Reader in) {
+ this(in, new char[8192]);
+ // 8192 matches the default buffer size of a BufferedReader so double
+ // buffering of the data is avoided.
+ }
+
+ public JSONParser(Reader in, char[] buffer) {
+ this.in = in;
+ this.buf = buffer;
+ }
+
+ // idea - if someone passes us a CharArrayReader, we could
+ // directly use that buffer as it's protected.
+
+ public JSONParser(char[] data, int start, int end) {
+ this.in = null;
+ this.buf = data;
+ this.start = start;
+ this.end = end;
+ }
+
+ public JSONParser(String data) {
+ this(data, 0, data.length());
+ }
+
+ public JSONParser(String data, int start, int end) {
+ this.in = null;
+ this.start = start;
+ this.end = end;
+ this.buf = new char[end - start];
+ data.getChars(start, end, buf, 0);
+ }
+
+ public int getFlags() {
+ return flags;
+ }
+
+ public int setFlags(int flags) {
+ int oldFlags = flags;
+ this.flags = flags;
+ return oldFlags;
+ }
+
+ // temporary output buffer
+ private final CharArr out = new CharArr(64);
+
+ // We need to keep some state in order to (at a minimum) know if
+ // we should skip ',' or ':'.
+ private byte[] stack = new byte[16];
+ private int ptr = 0; // pointer into the stack of parser states
+ private byte state = 0; // current parser state
+
+ // parser states stored in the stack
+ private static final byte DID_OBJSTART = 1; // '{' just read
+ private static final byte DID_ARRSTART = 2; // '[' just read
+ private static final byte DID_ARRELEM = 3; // array element just read
+ private static final byte DID_MEMNAME = 4; // object member name (map key) just read
+ private static final byte DID_MEMVAL = 5; // object member value (map val) just read
+
+ // info about value that was just read (or is in the middle of being read)
+ private int valstate;
+
+ // push current parser state (use at start of new container)
+ private final void push() {
+ if (ptr >= stack.length) {
+ // doubling here is probably overkill, but anything that needs to double more than
+ // once (32 levels deep) is very atypical anyway.
+ byte[] newstack = new byte[stack.length << 1];
+ System.arraycopy(stack, 0, newstack, 0, stack.length);
+ stack = newstack;
+ }
+ stack[ptr++] = state;
+ }
+
+ // pop parser state (use at end of container)
+ private final void pop() {
+ if (--ptr < 0) {
+ throw err("Unbalanced container");
+ } else {
+ state = stack[ptr];
+ }
+ }
+
+ protected void fill() throws IOException {
+ if (in != null) {
+ gpos += end;
+ start = 0;
+ int num = in.read(buf, 0, buf.length);
+ end = num >= 0 ? num : 0;
+ }
+ if (start >= end) eof = true;
+ }
+
+ private void getMore() throws IOException {
+ fill();
+ if (start >= end) {
+ throw err(null);
+ }
+ }
+
+ protected int getChar() throws IOException {
+ if (start >= end) {
+ fill();
+ if (start >= end) return -1;
+ }
+ return buf[start++];
+ }
+
+ /**
+ * Returns true if the given character is considered to be whitespace.
+ * One difference between Java's Character.isWhitespace() is that this method
+ * considers a hard space (non-breaking space, or nbsp) to be whitespace.
+ */
+ protected static final boolean isWhitespace(int ch) {
+ return (Character.isWhitespace(ch) || ch == 0x00a0);
+ }
+
+ private static final long WS_MASK = (1L << ' ') | (1L << '\t') | (1L << '\r') | (1L << '\n') | (1L << '#') | (1L << '/') | (0x01); // set 1 bit so 0xA0 will be flagged as whitespace
+
+ protected int getCharNWS() throws IOException {
+ for (; ; ) {
+ int ch = getChar();
+ // getCharNWS is normally called in the context of expecting certain JSON special characters
+ // such as ":}"],"
+ // all of these characters are below 64 (including comment chars '/' and '#', so we can make this the fast path
+ // even w/o checking the range first. We'll only get some false-positives while using bare strings (chars "IJMc")
+ if (((WS_MASK >> ch) & 0x01) == 0) {
+ return ch;
+ } else if (ch <= ' ') { // this will only be true if one of the whitespace bits was set
+ continue;
+ } else if (ch == '/') {
+ getSlashComment();
+ } else if (ch == '#') {
+ getNewlineComment();
+ } else if (!isWhitespace(ch)) { // we'll only reach here with certain bare strings, errors, or strange whitespace like 0xa0
+ return ch;
+ }
+
+ /***
+ // getCharNWS is normally called in the context of expecting certain JSON special characters
+ // such as ":}"],"
+ // all of these characters are below 64 (including comment chars '/' and '#', so we can make this the fast path
+ if (ch < 64) {
+ if (((WS_MASK >> ch) & 0x01) == 0) return ch;
+ if (ch <= ' ') continue; // whitespace below a normal space
+ if (ch=='/') {
+ getSlashComment();
+ } else if (ch=='#') {
+ getNewlineComment();
+ }
+ } else if (!isWhitespace(ch)) { // check for higher whitespace like 0xA0
+ return ch;
+ }
+ ***/
+
+ /** older code
+ switch (ch) {
+ case ' ' :
+ case '\t' :
+ case '\r' :
+ case '\n' :
+ continue outer;
+ case '#' :
+ getNewlineComment();
+ continue outer;
+ case '/' :
+ getSlashComment();
+ continue outer;
+ default:
+ return ch;
+ }
+ **/
+ }
+ }
+
+ protected int getCharNWS(int ch) throws IOException {
+ for (; ; ) {
+ // getCharNWS is normally called in the context of expecting certain JSON special characters
+ // such as ":}"],"
+ // all of these characters are below 64 (including comment chars '/' and '#', so we can make this the fast path
+ // even w/o checking the range first. We'll only get some false-positives while using bare strings (chars "IJMc")
+ if (((WS_MASK >> ch) & 0x01) == 0) {
+ return ch;
+ } else if (ch <= ' ') { // this will only be true if one of the whitespace bits was set
+ // whitespace... get new char at bottom of loop
+ } else if (ch == '/') {
+ getSlashComment();
+ } else if (ch == '#') {
+ getNewlineComment();
+ } else if (!isWhitespace(ch)) { // we'll only reach here with certain bare strings, errors, or strange whitespace like 0xa0
+ return ch;
+ }
+ ch = getChar();
+ }
+ }
+
+ protected int getCharExpected(int expected) throws IOException {
+ for (; ; ) {
+ int ch = getChar();
+ if (ch == expected) return expected;
+ if (ch == ' ') continue;
+ return getCharNWS(ch);
+ }
+ }
+
+ protected void getNewlineComment() throws IOException {
+ // read a # or a //, so go until newline
+ for (; ; ) {
+ int ch = getChar();
+ // don't worry about DOS /r/n... we'll stop on the \r and let the rest of the whitespace
+ // eater consume the \n
+ if (ch == '\n' || ch == '\r' || ch == -1) {
+ return;
+ }
+ }
+ }
+
+ protected void getSlashComment() throws IOException {
+ int ch = getChar();
+ if (ch == '/') {
+ getNewlineComment();
+ return;
+ }
+
+ if (ch != '*') {
+ throw err("Invalid comment: expected //, /*, or #");
+ }
+
+ ch = getChar();
+ for (; ; ) {
+ if (ch == '*') {
+ ch = getChar();
+ if (ch == '/') {
+ return;
+ } else if (ch == '*') {
+ // handle cases of *******/
+ continue;
+ }
+ }
+ if (ch == -1) {
+ return;
+ }
+ ch = getChar();
+ }
+ }
+
+
+ protected boolean matchBareWord(char[] arr) throws IOException {
+ for (int i = 1; i < arr.length; i++) {
+ int ch = getChar();
+ if (ch != arr[i]) {
+ if ((flags & ALLOW_UNQUOTED_STRING_VALUES) == 0) {
+ throw err("Expected " + new String(arr));
+ } else {
+ stringTerm = 0;
+ out.reset();
+ out.write(arr, 0, i);
+ if (!eof) {
+ start--;
+ }
+ return false;
+ }
+ }
+ }
+
+ // if we don't allow bare strings, we don't need to check that the string actually terminates... just
+ // let things fail as the parser tries to continue
+ if ((flags & ALLOW_UNQUOTED_STRING_VALUES) == 0) {
+ return true;
+ }
+
+ // check that the string actually terminates... for example trueX should return false
+ int ch = getChar();
+ if (eof) {
+ return true;
+ } else if (!isUnquotedStringChar(ch)) {
+ start--;
+ return true;
+ }
+
+ // we encountered something like "trueX" when matching "true"
+ stringTerm = 0;
+ out.reset();
+ out.unsafeWrite(arr, 0, arr.length);
+ out.unsafeWrite(ch);
+ return false;
+ }
+
+ protected ParseException err(String msg) {
+ // We can't tell if EOF was hit by comparing start<=end
+ // because the illegal char could have been the last in the buffer
+ // or in the stream. To deal with this, the "eof" var was introduced
+ if (!eof && start > 0) start--; // backup one char
+ String chs = "char=" + ((start >= end) ? "(EOF)" : "" + buf[start]);
+ String pos = "position=" + (gpos + start);
+ String tot = chs + ',' + pos + getContext();
+ if (msg == null) {
+ if (start >= end) msg = "Unexpected EOF";
+ else msg = "JSON Parse Error";
+ }
+ return new ParseException(msg + ": " + tot);
+ }
+
+ private String getContext() {
+ String context = "";
+ if (start >= 0) {
+ context += " AFTER='" + errEscape(Math.max(start - 60, 0), start + 1) + "'";
+ }
+ if (start < end) {
+ context += " BEFORE='" + errEscape(start + 1, start + 40) + "'";
+ }
+ return context;
+ }
+
+ private String errEscape(int a, int b) {
+ b = Math.min(b, end);
+ if (a >= b) return "";
+ return new String(buf, a, b - a).replaceAll("\\s+", " ");
+ }
+
+
+ private boolean bool; // boolean value read
+ private long lval; // long value read
+ private int nstate; // current state while reading a number
+ private static final int HAS_FRACTION = 0x01; // nstate flag, '.' already read
+ private static final int HAS_EXPONENT = 0x02; // nstate flag, '[eE][+-]?[0-9]' already read
+
+ /**
+ * Returns the long read... only significant if valstate==LONG after
+ * this call. firstChar should be the first numeric digit read.
+ */
+ private long readNumber(int firstChar, boolean isNeg) throws IOException {
+ out.unsafeWrite(firstChar); // unsafe OK since we know output is big enough
+ // We build up the number in the negative plane since it's larger (by one) than
+ // the positive plane.
+ long v = '0' - firstChar;
+ // can't overflow a long in 18 decimal digits (i.e. 17 additional after the first).
+ // we also need 22 additional to handle double so we'll handle in 2 separate loops.
+ int i;
+ for (i = 0; i < 17; i++) {
+ int ch = getChar();
+ // TODO: is this switch faster as an if-then-else?
+ switch (ch) {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ v = v * 10 - (ch - '0');
+ out.unsafeWrite(ch);
+ continue;
+ case '.':
+ out.unsafeWrite('.');
+ valstate = readFrac(out, 22 - i);
+ return 0;
+ case 'e':
+ case 'E':
+ out.unsafeWrite(ch);
+ nstate = 0;
+ valstate = readExp(out, 22 - i);
+ return 0;
+ default:
+ // return the number, relying on nextEvent() to return an error
+ // for invalid chars following the number.
+ if (ch != -1) --start; // push back last char if not EOF
+
+ valstate = LONG;
+ return isNeg ? v : -v;
+ }
+ }
+
+ // after this, we could overflow a long and need to do extra checking
+ boolean overflow = false;
+ long maxval = isNeg ? Long.MIN_VALUE : -Long.MAX_VALUE;
+
+ for (; i < 22; i++) {
+ int ch = getChar();
+ switch (ch) {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ if (v < (0x8000000000000000L / 10)) overflow = true; // can't multiply by 10 w/o overflowing
+ v *= 10;
+ int digit = ch - '0';
+ if (v < maxval + digit) overflow = true; // can't add digit w/o overflowing
+ v -= digit;
+ out.unsafeWrite(ch);
+ continue;
+ case '.':
+ out.unsafeWrite('.');
+ valstate = readFrac(out, 22 - i);
+ return 0;
+ case 'e':
+ case 'E':
+ out.unsafeWrite(ch);
+ nstate = 0;
+ valstate = readExp(out, 22 - i);
+ return 0;
+ default:
+ // return the number, relying on nextEvent() to return an error
+ // for invalid chars following the number.
+ if (ch != -1) --start; // push back last char if not EOF
+
+ valstate = overflow ? BIGNUMBER : LONG;
+ return isNeg ? v : -v;
+ }
+ }
+
+
+ nstate = 0;
+ valstate = BIGNUMBER;
+ return 0;
+ }
+
+
+ // read digits right of decimal point
+ private int readFrac(CharArr arr, int lim) throws IOException {
+ nstate = HAS_FRACTION; // deliberate set instead of '|'
+ while (--lim >= 0) {
+ int ch = getChar();
+ if (ch >= '0' && ch <= '9') {
+ arr.write(ch);
+ } else if (ch == 'e' || ch == 'E') {
+ arr.write(ch);
+ return readExp(arr, lim);
+ } else {
+ if (ch != -1) start--; // back up
+ return NUMBER;
+ }
+ }
+ return BIGNUMBER;
+ }
+
+
+ // call after 'e' or 'E' has been seen to read the rest of the exponent
+ private int readExp(CharArr arr, int lim) throws IOException {
+ nstate |= HAS_EXPONENT;
+ int ch = getChar();
+ lim--;
+
+ if (ch == '+' || ch == '-') {
+ arr.write(ch);
+ ch = getChar();
+ lim--;
+ }
+
+ // make sure at least one digit is read.
+ if (ch < '0' || ch > '9') {
+ throw err("missing exponent number");
+ }
+ arr.write(ch);
+
+ return readExpDigits(arr, lim);
+ }
+
+ // continuation of readExpStart
+ private int readExpDigits(CharArr arr, int lim) throws IOException {
+ while (--lim >= 0) {
+ int ch = getChar();
+ if (ch >= '0' && ch <= '9') {
+ arr.write(ch);
+ } else {
+ if (ch != -1) start--; // back up
+ return NUMBER;
+ }
+ }
+ return BIGNUMBER;
+ }
+
+ private void continueNumber(CharArr arr) throws IOException {
+ if (arr != out) arr.write(out);
+
+ if ((nstate & HAS_EXPONENT) != 0) {
+ readExpDigits(arr, Integer.MAX_VALUE);
+ return;
+ }
+ if (nstate != 0) {
+ readFrac(arr, Integer.MAX_VALUE);
+ return;
+ }
+
+ for (; ; ) {
+ int ch = getChar();
+ if (ch >= '0' && ch <= '9') {
+ arr.write(ch);
+ } else if (ch == '.') {
+ arr.write(ch);
+ readFrac(arr, Integer.MAX_VALUE);
+ return;
+ } else if (ch == 'e' || ch == 'E') {
+ arr.write(ch);
+ readExp(arr, Integer.MAX_VALUE);
+ return;
+ } else {
+ if (ch != -1) start--;
+ return;
+ }
+ }
+ }
+
+
+ private int hexval(int hexdig) {
+ if (hexdig >= '0' && hexdig <= '9') {
+ return hexdig - '0';
+ } else if (hexdig >= 'A' && hexdig <= 'F') {
+ return hexdig + (10 - 'A');
+ } else if (hexdig >= 'a' && hexdig <= 'f') {
+ return hexdig + (10 - 'a');
+ }
+ throw err("invalid hex digit");
+ }
+
+ // backslash has already been read when this is called
+ private char readEscapedChar() throws IOException {
+ int ch = getChar();
+ switch (ch) {
+ case '"':
+ return '"';
+ case '\'':
+ return '\'';
+ case '\\':
+ return '\\';
+ case '/':
+ return '/';
+ case 'n':
+ return '\n';
+ case 'r':
+ return '\r';
+ case 't':
+ return '\t';
+ case 'f':
+ return '\f';
+ case 'b':
+ return '\b';
+ case 'u':
+ return (char) (
+ (hexval(getChar()) << 12)
+ | (hexval(getChar()) << 8)
+ | (hexval(getChar()) << 4)
+ | (hexval(getChar())));
+ }
+ if ((flags & ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER) != 0 && ch != EOF) {
+ return (char) ch;
+ }
+ throw err("Invalid character escape");
+ }
+
+ // a dummy buffer we can use to point at other buffers
+ private final CharArr tmp = new CharArr(null, 0, 0);
+
+ private CharArr readStringChars() throws IOException {
+ if (stringTerm == 0) {
+ // "out" will already contain the first part of the bare string, so don't reset it
+ readStringBare(out);
+ return out;
+ }
+
+ char terminator = (char) stringTerm;
+ int i;
+ for (i = start; i < end; i++) {
+ char c = buf[i];
+ if (c == terminator) {
+ tmp.set(buf, start, i); // directly use input buffer
+ start = i + 1; // advance past last '"'
+ return tmp;
+ } else if (c == '\\') {
+ break;
+ }
+ }
+ out.reset();
+ readStringChars2(out, i);
+ return out;
+ }
+
+
+ // middle is the pointer to the middle of a buffer to start scanning for a non-string
+ // character ('"' or "/"). start<=middle<end
+ // this should be faster for strings with fewer escapes, but probably slower for many escapes.
+ private void readStringChars2(CharArr arr, int middle) throws IOException {
+ if (stringTerm == 0) {
+ readStringBare(arr);
+ return;
+ }
+
+ char terminator = (char) stringTerm;
+
+ for (; ; ) {
+ if (middle >= end) {
+ arr.write(buf, start, middle - start);
+ start = middle;
+ getMore();
+ middle = start;
+ }
+ int ch = buf[middle++];
+ if (ch == terminator) {
+ int len = middle - start - 1;
+ if (len > 0) arr.write(buf, start, len);
+ start = middle;
+ return;
+ } else if (ch == '\\') {
+ int len = middle - start - 1;
+ if (len > 0) arr.write(buf, start, len);
+ start = middle;
+ arr.write(readEscapedChar());
+ middle = start;
+ }
+ }
+ }
+
+ private void readStringBare(CharArr arr) throws IOException {
+ if (arr != out) {
+ arr.append(out);
+ }
+
+ for (; ; ) {
+ int ch = getChar();
+ if (!isUnquotedStringChar(ch)) {
+ if (ch == -1) break;
+ if (ch == '\\') {
+ arr.write(readEscapedChar());
+ continue;
+ }
+ start--;
+ break;
+ }
+
+ if (ch == '\\') {
+ arr.write(readEscapedChar());
+ continue;
+ }
+
+ arr.write(ch);
+ }
+ }
+
+
+ // isName==true if this is a field name (as opposed to a value)
+ protected void handleNonDoubleQuoteString(int ch, boolean isName) throws IOException {
+ if (ch == '\'') {
+ stringTerm = ch;
+ if ((flags & ALLOW_SINGLE_QUOTES) == 0) {
+ throw err("Single quoted strings not allowed");
+ }
+ } else {
+ if (isName && (flags & ALLOW_UNQUOTED_KEYS) == 0
+ || !isName && (flags & ALLOW_UNQUOTED_STRING_VALUES) == 0
+ || eof) {
+ if (isName) {
+ throw err("Expected quoted string");
+ } else {
+ throw err(null);
+ }
+ }
+
+ if (!isUnquotedStringStart(ch)) {
+ throw err(null);
+ }
+
+ stringTerm = 0; // signal for unquoted string
+ out.reset();
+ out.unsafeWrite(ch);
+ }
+ }
+
+ private static boolean isUnquotedStringStart(int ch) {
+ return Character.isJavaIdentifierStart(ch);
+ }
+
+ // What characters are allowed to continue an unquoted string
+ // once we know we are in one.
+ private static boolean isUnquotedStringChar(int ch) {
+ return Character.isJavaIdentifierPart(ch)
+ || ch == '.'
+ || ch == '-'
+ || ch == '/';
+
+ // would checking for a-z first speed up the common case?
+
+ // possibly much more liberal unquoted string handling...
+ /***
+ switch (ch) {
+ case -1:
+ case ' ':
+ case '\t':
+ case '\r':
+ case '\n':
+ case '}':
+ case ']':
+ case ',':
+ case ':':
+ case '=': // reserved for future use
+ case '\\': // check for backslash should come after this function call
+ return false;
+ }
+ return true;
+ ***/
+ }
+
+
+ /*** alternate implementation
+ // middle is the pointer to the middle of a buffer to start scanning for a non-string
+ // character ('"' or "/"). start<=middle<end
+ private void readStringChars2a(CharArr arr, int middle) throws IOException {
+ int ch=0;
+ for(;;) {
+ // find the next non-string char
+ for (; middle<end; middle++) {
+ ch = buf[middle];
+ if (ch=='"' || ch=='\\') break;
+ }
+
+ arr.write(buf,start,middle-start);
+ if (middle>=end) {
+ getMore();
+ middle=start;
+ } else {
+ start = middle+1; // set buffer pointer to correct spot
+ if (ch=='"') {
+ valstate=0;
+ return;
+ } else if (ch=='\\') {
+ arr.write(readEscapedChar());
+ if (start>=end) getMore();
+ middle=start;
+ }
+ }
+ }
+ }
+ ***/
+
+
+ // return the next event when parser is in a neutral state (no
+ // map separators or array element separators to read
+ private int next(int ch) throws IOException {
+ // TODO: try my own form of indirect jump... look up char class and index directly into handling implementation?
+ for (; ; ) {
+ switch (ch) {
+ case ' ': // this is not the exclusive list of whitespace chars... the rest are handled in default:
+ case '\t':
+ case '\r':
+ case '\n':
+ ch = getCharNWS(); // calling getCharNWS here seems faster than letting the switch handle it
+ break;
+ case '"':
+ stringTerm = '"';
+ valstate = STRING;
+ return STRING;
+ case '\'':
+ if ((flags & ALLOW_SINGLE_QUOTES) == 0) {
+ throw err("Single quoted strings not allowed");
+ }
+ stringTerm = '\'';
+ valstate = STRING;
+ return STRING;
+ case '{':
+ push();
+ state = DID_OBJSTART;
+ return OBJECT_START;
+ case '[':
+ push();
+ state = DID_ARRSTART;
+ return ARRAY_START;
+ case '0':
+ out.reset();
+ //special case '0'? If next char isn't '.' val=0
+ ch = getChar();
+ if (ch == '.') {
+ start--;
+ ch = '0';
+ readNumber('0', false);
+ return valstate;
+ } else if (ch > '9' || ch < '0') {
+ out.unsafeWrite('0');
+ if (ch != -1) start--;
+ lval = 0;
+ valstate = LONG;
+ return LONG;
+ } else {
+ throw err("Leading zeros not allowed");
+ }
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ out.reset();
+ lval = readNumber(ch, false);
+ return valstate;
+ case '-':
+ out.reset();
+ out.unsafeWrite('-');
+ ch = getChar();
+ if (ch < '0' || ch > '9') throw err("expected digit after '-'");
+ lval = readNumber(ch, true);
+ return valstate;
+ case 't':
+ // TODO: test performance of this non-branching inline version.
+ // if ((('r'-getChar())|('u'-getChar())|('e'-getChar())) != 0) throw err("");
+ if (matchBareWord(JSONUtil.TRUE_CHARS)) {
+ bool = true;
+ valstate = BOOLEAN;
+ return valstate;
+ } else {
+ valstate = STRING;
+ return STRING;
+ }
+ case 'f':
+ if (matchBareWord(JSONUtil.FALSE_CHARS)) {
+ bool = false;
+ valstate = BOOLEAN;
+ return valstate;
+ } else {
+ valstate = STRING;
+ return STRING;
+ }
+ case 'n':
+ if (matchBareWord(JSONUtil.NULL_CHARS)) {
+ valstate = NULL;
+ return valstate;
+ } else {
+ valstate = STRING;
+ return STRING;
+ }
+ case '/':
+ getSlashComment();
+ ch = getChar();
+ break;
+ case '#':
+ getNewlineComment();
+ ch = getChar();
+ break;
+ case ']': // This only happens with a trailing comma (or an error)
+ if (state != DID_ARRELEM || (flags & ALLOW_EXTRA_COMMAS) == 0) {
+ throw err("Unexpected array closer ]");
+ }
+ pop();
+ return event = ARRAY_END;
+ case '}': // This only happens with a trailing comma (or an error)
+ if (state != DID_MEMVAL || (flags & ALLOW_EXTRA_COMMAS) == 0) {
+ throw err("Unexpected object closer }");
+ }
+ pop();
+ return event = ARRAY_END;
+ case ',': // This only happens with input like [1,]
+ if ((state != DID_ARRELEM && state != DID_MEMVAL) || (flags & ALLOW_EXTRA_COMMAS) == 0) {
+ throw err("Unexpected comma");
+ }
+ ch = getChar();
+ break;
+ case -1:
+ if (getLevel() > 0) throw err("Premature EOF");
+ return EOF;
+ default:
+ // Handle unusual unicode whitespace like no-break space (0xA0)
+ if (isWhitespace(ch)) {
+ ch = getChar(); // getCharNWS() would also work
+ break;
+ }
+ handleNonDoubleQuoteString(ch, false);
+ valstate = STRING;
+ return STRING;
+ // throw err(null);
+ }
+
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "start=" + start + ",end=" + end + ",state=" + state + "valstate=" + valstate;
+ }
+
+
+ /**
+ * Returns the next event encountered in the JSON stream, one of
+ * <ul>
+ * <li>{@link #STRING}</li>
+ * <li>{@link #LONG}</li>
+ * <li>{@link #NUMBER}</li>
+ * <li>{@link #BIGNUMBER}</li>
+ * <li>{@link #BOOLEAN}</li>
+ * <li>{@link #NULL}</li>
+ * <li>{@link #OBJECT_START}</li>
+ * <li>{@link #OBJECT_END}</li>
+ * <li>{@link #OBJECT_END}</li>
+ * <li>{@link #ARRAY_START}</li>
+ * <li>{@link #ARRAY_END}</li>
+ * <li>{@link #EOF}</li>
+ * </ul>
+ */
+ public int nextEvent() throws IOException {
+ if (valstate != 0) {
+ if (valstate == STRING) {
+ readStringChars2(devNull, start);
+ } else if (valstate == BIGNUMBER) {
+ continueNumber(devNull);
+ }
+ valstate = 0;
+ }
+
+ int ch;
+ outer:
+ for (; ; ) {
+ switch (state) {
+ case 0:
+ event = next(getChar());
+ if (event == STRING && (flags & OPTIONAL_OUTER_BRACES) != 0) {
+ if (start > 0) start--;
+ missingOpeningBrace = true;
+ stringTerm = 0;
+ valstate = 0;
+ event = next('{');
+ }
+ return event;
+ case DID_OBJSTART:
+ ch = getCharExpected('"');
+ if (ch == '}') {
+ pop();
+ return event = OBJECT_END;
+ }
+ if (ch == '"') {
+ stringTerm = ch;
+ } else if (ch == ',' && (flags & ALLOW_EXTRA_COMMAS) != 0) {
+ continue outer;
+ } else {
+ handleNonDoubleQuoteString(ch, true);
+ }
+ state = DID_MEMNAME;
+ valstate = STRING;
+ return event = STRING;
+ case DID_MEMNAME:
+ ch = getCharExpected(':');
+ if (ch != ':') {
+ if ((ch == '{' || ch == '[') && (flags & ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT) != 0) {
+ start--;
+ } else {
+ throw err("Expected key,value separator ':'");
+ }
+ }
+ state = DID_MEMVAL; // set state first because it might be pushed...
+ return event = next(getChar());
+ case DID_MEMVAL:
+ ch = getCharExpected(',');
+ if (ch == '}') {
+ pop();
+ return event = OBJECT_END;
+ } else if (ch != ',') {
+ if ((flags & ALLOW_EXTRA_COMMAS) != 0 && (ch == '\'' || ch == '"' || Character.isLetter(ch))) {
+ start--;
+ } else if (missingOpeningBrace && ch == -1 && (flags & OPTIONAL_OUTER_BRACES) != 0) {
+ missingOpeningBrace = false;
+ pop();
+ return event = OBJECT_END;
+ } else throw err("Expected ',' or '}'");
+ }
+ ch = getCharExpected('"');
+ if (ch == '"') {
+ stringTerm = ch;
+ } else if ((ch == ',' || ch == '}') && (flags & ALLOW_EXTRA_COMMAS) != 0) {
+ if (ch == ',') continue outer;
+ pop();
+ return event = OBJECT_END;
+ } else {
+ handleNonDoubleQuoteString(ch, true);
+ }
+ state = DID_MEMNAME;
+ valstate = STRING;
+ return event = STRING;
+ case DID_ARRSTART:
+ ch = getCharNWS();
+ if (ch == ']') {
+ pop();
+ return event = ARRAY_END;
+ }
+ state = DID_ARRELEM; // set state first, might be pushed...
+ return event = next(ch);
+ case DID_ARRELEM:
+ ch = getCharExpected(',');
+ if (ch == ',') {
+ // state = DID_ARRELEM; // redundant
+ return event = next(getChar());
+ } else if (ch == ']') {
+ pop();
+ return event = ARRAY_END;
+ } else {
+ if ((ch == '{' || ch == '[') && (flags & ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT) != 0) {
+ return event = next(ch);
+ } else {
+ throw err("Expected ',' or ']'");
+ }
+ }
+ }
+ } // end for(;;)
+ }
+
+ public int lastEvent() {
+ return event;
+ }
+
+ public boolean wasKey() {
+ return state == DID_MEMNAME;
+ }
+
+
+ private void goTo(int what) throws IOException {
+ if (valstate == what) {
+ valstate = 0;
+ return;
+ }
+ if (valstate == 0) {
+ /*int ev = */
+ nextEvent(); // TODO
+ if (valstate != what) {
+ throw err("type mismatch");
+ }
+ valstate = 0;
+ } else {
+ throw err("type mismatch");
+ }
+ }
+
+ /**
+ * Returns the JSON string value, decoding any escaped characters.
+ */
+ public String getString() throws IOException {
+ return getStringChars().toString();
+ }
+
+ /**
+ * Returns the characters of a JSON string value, decoding any escaped characters.
+ * The underlying buffer of the returned <code>CharArr</code> should *not* be
+ * modified as it may be shared with the input buffer.
+ * The returned <code>CharArr</code> will only be valid up until
+ * the next JSONParser method is called. Any required data should be
+ * read before that point.
+ */
+ public CharArr getStringChars() throws IOException {
+ goTo(STRING);
+ return readStringChars();
+ }
+
+ /**
+ * Reads a JSON string into the output, decoding any escaped characters.
+ */
+ public void getString(CharArr output) throws IOException {
+ goTo(STRING);
+ readStringChars2(output, start);
+ }
+
+ /**
+ * Reads a number from the input stream and parses it as a long, only if
+ * the value will in fact fit into a signed 64 bit integer.
+ */
+ public long getLong() throws IOException {
+ goTo(LONG);
+ return lval;
+ }
+
+ /**
+ * Reads a number from the input stream and parses it as a double
+ */
+ public double getDouble() throws IOException {
+ return Double.parseDouble(getNumberChars().toString());
+ }
+
+ /**
+ * Returns the characters of a JSON numeric value.
+ * <p>The underlying buffer of the returned <code>CharArr</code> should *not* be
+ * modified as it may be shared with the input buffer.
+ * <p>The returned <code>CharArr</code> will only be valid up until
+ * the next JSONParser method is called. Any required data should be
+ * read before that point.
+ */
+ public CharArr getNumberChars() throws IOException {
+ int ev = 0;
+ if (valstate == 0) ev = nextEvent();
+
+ if (valstate == LONG || valstate == NUMBER) {
+ valstate = 0;
+ return out;
+ } else if (valstate == BIGNUMBER) {
+ continueNumber(out);
+ valstate = 0;
+ return out;
+ } else {
+ throw err("Unexpected " + ev);
+ }
+ }
+
+ /**
+ * Reads a JSON numeric value into the output.
+ */
+ public void getNumberChars(CharArr output) throws IOException {
+ int ev = 0;
+ if (valstate == 0) ev = nextEvent();
+ if (valstate == LONG || valstate == NUMBER) output.write(this.out);
+ else if (valstate == BIGNUMBER) {
+ continueNumber(output);
+ } else {
+ throw err("Unexpected " + ev);
+ }
+ valstate = 0;
+ }
+
+ /**
+ * Reads a boolean value
+ */
+ public boolean getBoolean() throws IOException {
+ goTo(BOOLEAN);
+ return bool;
+ }
+
+ /**
+ * Reads a null value
+ */
+ public void getNull() throws IOException {
+ goTo(NULL);
+ }
+
+ /**
+ * @return the current nesting level, the number of parent objects or arrays.
+ */
+ public int getLevel() {
+ return ptr;
+ }
+
+ public long getPosition() {
+ return gpos + start;
+ }
+}
\ No newline at end of file
diff --git a/solr/solrj/src/java/org/noggit/JSONUtil.java b/solr/solrj/src/java/org/noggit/JSONUtil.java
new file mode 100644
index 0000000..4c74759
--- /dev/null
+++ b/solr/solrj/src/java/org/noggit/JSONUtil.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.noggit;
+
+
+public class JSONUtil {
+ public static final char[] TRUE_CHARS = new char[]{'t', 'r', 'u', 'e'};
+ public static final char[] FALSE_CHARS = new char[]{'f', 'a', 'l', 's', 'e'};
+ public static final char[] NULL_CHARS = new char[]{'n', 'u', 'l', 'l'};
+ public static final char[] HEX_CHARS = new char[]{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+ public static final char VALUE_SEPARATOR = ',';
+ public static final char NAME_SEPARATOR = ':';
+ public static final char OBJECT_START = '{';
+ public static final char OBJECT_END = '}';
+ public static final char ARRAY_START = '[';
+ public static final char ARRAY_END = ']';
+
+ public static String toJSON(Object o) {
+ CharArr out = new CharArr();
+ new JSONWriter(out).write(o);
+ return out.toString();
+ }
+
+ /**
+ * @param o The object to convert to JSON
+ * @param indentSize The number of space characters to use as an indent (default 2). 0=newlines but no spaces, -1=no indent at all.
+ */
+ public static String toJSON(Object o, int indentSize) {
+ CharArr out = new CharArr();
+ new JSONWriter(out, indentSize).write(o);
+ return out.toString();
+ }
+
+ public static void writeNumber(int number, CharArr out) {
+ out.write(Integer.toString(number));
+ }
+
+ public static void writeNumber(long number, CharArr out) {
+ out.write(Long.toString(number));
+ }
+
+ public static void writeNumber(float number, CharArr out) {
+ out.write(Float.toString(number));
+ }
+
+ public static void writeNumber(double number, CharArr out) {
+ out.write(Double.toString(number));
+ }
+
+ public static void writeString(CharArr val, CharArr out) {
+ writeString(val.getArray(), val.getStart(), val.getEnd(), out);
+ }
+
+ public static void writeString(char[] val, int start, int end, CharArr out) {
+ out.write('"');
+ writeStringPart(val, start, end, out);
+ out.write('"');
+ }
+
+ public static void writeString(String val, int start, int end, CharArr out) {
+ out.write('"');
+ writeStringPart(val, start, end, out);
+ out.write('"');
+ }
+
+ public static void writeString(CharSequence val, int start, int end, CharArr out) {
+ out.write('"');
+ writeStringPart(val, start, end, out);
+ out.write('"');
+ }
+
+ public static void writeStringPart(char[] val, int start, int end, CharArr out) {
+ for (int i = start; i < end; i++) {
+ char ch = val[i];
+ // When ch>=1f, (ch*146087937)&0xd6a01f80) is 0 only for characters that need escaping: " \\ u2028 u2029
+ // and has 7 false positives: 204a 4051 802f c022 c044 e04a e04b
+ if (ch > 0x1f && ((ch * 146087937) & 0xd6a01f80) != 0) {
+ out.write(ch);
+ } else {
+ writeChar(ch, out);
+ }
+ }
+ }
+
+ public static void writeChar(char ch, CharArr out) {
+ switch (ch) {
+ case '"':
+ case '\\':
+ out.write('\\');
+ out.write(ch);
+ break;
+ case '\r':
+ out.write('\\');
+ out.write('r');
+ break;
+ case '\n':
+ out.write('\\');
+ out.write('n');
+ break;
+ case '\t':
+ out.write('\\');
+ out.write('t');
+ break;
+ case '\b':
+ out.write('\\');
+ out.write('b');
+ break;
+ case '\f':
+ out.write('\\');
+ out.write('f');
+ break;
+ // case '/':
+ case '\u2028': // valid JSON, but not valid json script
+ case '\u2029':
+ unicodeEscape(ch, out);
+ break;
+ default:
+ if (ch <= 0x1F) {
+ unicodeEscape(ch, out);
+ } else {
+ out.write(ch);
+ }
+ }
+ }
+
+
+ public static void writeStringPart(String chars, int start, int end, CharArr out) {
+ // TODO: write in chunks?
+
+ int toWrite = end - start;
+ char[] arr = out.getArray();
+ int pos = out.getEnd();
+ int space = arr.length - pos;
+ if (space < toWrite) {
+ writeStringPart((CharSequence) chars, start, end, out);
+ return;
+ }
+
+ // get chars directly from String into output array
+ chars.getChars(start, end, arr, pos);
+
+ int endInOut = pos + toWrite;
+ out.setEnd(endInOut);
+ for (int i = pos; i < endInOut; i++) {
+ char ch = arr[i];
+
+ // When ch>=1f, (ch*146087937)&0xd6a01f80) is 0 only for characters that need escaping: " \\ u2028 u2029
+ // and has 7 false positives: 204a 4051 802f c022 c044 e04a e04b
+ if (ch <= 0x1f || ((ch * 146087937) & 0xd6a01f80) == 0) {
+ // We hit a char that needs escaping. do the rest char by char.
+ out.setEnd(i);
+ writeStringPart((CharSequence) chars, start + (i - pos), end, out);
+ return;
+ }
+ }
+ }
+
+ public static void writeStringPart(CharSequence chars, int start, int end, CharArr out) {
+ for (int i = start; i < end; i++) {
+ char ch = chars.charAt(i);
+ // When ch>=1f, (ch*146087937)&0xd6a01f80) is 0 only for characters that need escaping: " \\ u2028 u2029
+ // and has 7 false positives: 204a 4051 802f c022 c044 e04a e04b
+ if (ch > 0x1f && ((ch * 146087937) & 0xd6a01f80) != 0) {
+ out.write(ch);
+ } else {
+ writeChar(ch, out);
+ }
+ }
+ }
+
+
+ public static void unicodeEscape(int ch, CharArr out) {
+ out.write('\\');
+ out.write('u');
+ out.write(HEX_CHARS[ch >>> 12]);
+ out.write(HEX_CHARS[(ch >>> 8) & 0xf]);
+ out.write(HEX_CHARS[(ch >>> 4) & 0xf]);
+ out.write(HEX_CHARS[ch & 0xf]);
+ }
+
+ public static void writeNull(CharArr out) {
+ out.write(NULL_CHARS);
+ }
+
+ public static void writeBoolean(boolean val, CharArr out) {
+ out.write(val ? TRUE_CHARS : FALSE_CHARS);
+ }
+
+}
\ No newline at end of file
diff --git a/solr/solrj/src/java/org/noggit/JSONWriter.java b/solr/solrj/src/java/org/noggit/JSONWriter.java
new file mode 100644
index 0000000..dfec390
--- /dev/null
+++ b/solr/solrj/src/java/org/noggit/JSONWriter.java
@@ -0,0 +1,358 @@
+/*
+ * Copyright 2006- Yonik Seeley
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.util.*;
+
+
+public class JSONWriter {
+
+ /**
+ * Implement this interface on your class to support serialization
+ */
+ public static interface Writable {
+ public void write(JSONWriter writer);
+ }
+
+ protected int level;
+ protected int indent;
+ protected final CharArr out;
+
+ /**
+ * @param out the CharArr to write the output to.
+ * @param indentSize The number of space characters to use as an indent (default 2). 0=newlines but no spaces, -1=no indent at all.
+ */
+ public JSONWriter(CharArr out, int indentSize) {
+ this.out = out;
+ this.indent = indentSize;
+ }
+
+ public JSONWriter(CharArr out) {
+ this(out, 2);
+ }
+
+ public void setIndentSize(int indentSize) {
+ this.indent = indentSize;
+ }
+
+ public void indent() {
+ if (indent >= 0) {
+ out.write('\n');
+ if (indent > 0) {
+ int spaces = level * indent;
+ out.reserve(spaces);
+ for (int i = 0; i < spaces; i++) {
+ out.unsafeWrite(' ');
+ }
+ }
+ }
+ }
+
+ public void write(Object o) {
+ // NOTE: an instance-of chain was about 50% faster than hashing on the classes, even with perfect hashing.
+ if (o == null) {
+ writeNull();
+ } else if (o instanceof String) {
+ writeString((String) o);
+ } else if (o instanceof Number) {
+ if (o instanceof Integer || o instanceof Long) {
+ write(((Number) o).longValue());
+ } else if (o instanceof Float || o instanceof Double) {
+ write(((Number) o).doubleValue());
+ } else {
+ CharArr arr = new CharArr();
+ arr.write(o.toString());
+ writeNumber(arr);
+ }
+ } else if (o instanceof Map) {
+ write((Map<?, ?>) o);
+ } else if (o instanceof Collection) {
+ write((Collection<?>) o);
+ } else if (o instanceof Boolean) {
+ write(((Boolean) o).booleanValue());
+ } else if (o instanceof CharSequence) {
+ writeString((CharSequence) o);
+ } else if (o instanceof Writable) {
+ ((Writable) o).write(this);
+ } else if (o instanceof Object[]) {
+ write(Arrays.asList((Object[]) o));
+ } else if (o instanceof int[]) {
+ write((int[]) o);
+ } else if (o instanceof float[]) {
+ write((float[]) o);
+ } else if (o instanceof long[]) {
+ write((long[]) o);
+ } else if (o instanceof double[]) {
+ write((double[]) o);
+ } else if (o instanceof short[]) {
+ write((short[]) o);
+ } else if (o instanceof boolean[]) {
+ write((boolean[]) o);
+ } else if (o instanceof char[]) {
+ write((char[]) o);
+ } else if (o instanceof byte[]) {
+ write((byte[]) o);
+ } else {
+ handleUnknownClass(o);
+ }
+ }
+
+ /**
+ * Override this method for custom handling of unknown classes. Also see the Writable interface.
+ */
+ public void handleUnknownClass(Object o) {
+ writeString(o.toString());
+ }
+
+ public void write(Map<?, ?> val) {
+ startObject();
+ int sz = val.size();
+ boolean first = true;
+ for (Map.Entry<?, ?> entry : val.entrySet()) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ if (sz > 1) indent();
+ writeString(entry.getKey().toString());
+ writeNameSeparator();
+ write(entry.getValue());
+ }
+ endObject();
+ }
+
+ public void write(Collection<?> val) {
+ startArray();
+ int sz = val.size();
+ boolean first = true;
+ for (Object o : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ if (sz > 1) indent();
+ write(o);
+ }
+ endArray();
+ }
+
+ /**
+ * A byte[] may be either a single logical value, or a list of small integers.
+ * It's up to the implementation to decide.
+ */
+ public void write(byte[] val) {
+ startArray();
+ boolean first = true;
+ for (short v : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ write(v);
+ }
+ endArray();
+ }
+
+ public void write(short[] val) {
+ startArray();
+ boolean first = true;
+ for (short v : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ write(v);
+ }
+ endArray();
+ }
+
+ public void write(int[] val) {
+ startArray();
+ boolean first = true;
+ for (int v : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ write(v);
+ }
+ endArray();
+ }
+
+ public void write(long[] val) {
+ startArray();
+ boolean first = true;
+ for (long v : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ write(v);
+ }
+ endArray();
+ }
+
+ public void write(float[] val) {
+ startArray();
+ boolean first = true;
+ for (float v : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ write(v);
+ }
+ endArray();
+ }
+
+ public void write(double[] val) {
+ startArray();
+ boolean first = true;
+ for (double v : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ write(v);
+ }
+ endArray();
+ }
+
+ public void write(boolean[] val) {
+ startArray();
+ boolean first = true;
+ for (boolean v : val) {
+ if (first) {
+ first = false;
+ } else {
+ writeValueSeparator();
+ }
+ write(v);
+ }
+ endArray();
+ }
+
+
+ public void write(short number) {
+ write((int) number);
+ }
+
+ public void write(byte number) {
+ write((int) number);
+ }
+
+
+ public void writeNull() {
+ JSONUtil.writeNull(out);
+ }
+
+ public void writeString(String str) {
+ JSONUtil.writeString(str, 0, str.length(), out);
+ }
+
+ public void writeString(CharSequence str) {
+ JSONUtil.writeString(str, 0, str.length(), out);
+ }
+
+ public void writeString(CharArr str) {
+ JSONUtil.writeString(str, out);
+ }
+
+ public void writeStringStart() {
+ out.write('"');
+ }
+
+ public void writeStringChars(CharArr partialStr) {
+ JSONUtil.writeStringPart(partialStr.getArray(), partialStr.getStart(), partialStr.getEnd(), out);
+ }
+
+ public void writeStringEnd() {
+ out.write('"');
+ }
+
+ public void write(long number) {
+ JSONUtil.writeNumber(number, out);
+ }
+
+ public void write(int number) {
+ JSONUtil.writeNumber(number, out);
+ }
+
+ public void write(double number) {
+ JSONUtil.writeNumber(number, out);
+ }
+
+ public void write(float number) {
+ JSONUtil.writeNumber(number, out);
+ }
+
+ public void write(boolean bool) {
+ JSONUtil.writeBoolean(bool, out);
+ }
+
+ public void write(char[] val) {
+ JSONUtil.writeString(val, 0, val.length, out);
+ }
+
+ public void writeNumber(CharArr digits) {
+ out.write(digits);
+ }
+
+ public void writePartialNumber(CharArr digits) {
+ out.write(digits);
+ }
+
+ public void startObject() {
+ out.write('{');
+ level++;
+ }
+
+ public void endObject() {
+ out.write('}');
+ level--;
+ }
+
+ public void startArray() {
+ out.write('[');
+ level++;
+ }
+
+ public void endArray() {
+ out.write(']');
+ level--;
+ }
+
+ public void writeValueSeparator() {
+ out.write(',');
+ }
+
+ public void writeNameSeparator() {
+ out.write(':');
+ }
+
+}
diff --git a/solr/solrj/src/java/org/noggit/ObjectBuilder.java b/solr/solrj/src/java/org/noggit/ObjectBuilder.java
new file mode 100644
index 0000000..945a96b
--- /dev/null
+++ b/solr/solrj/src/java/org/noggit/ObjectBuilder.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2006- Yonik Seeley
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+
+import java.util.*;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+public class ObjectBuilder {
+
+ public static Object fromJSON(String json) throws IOException {
+ JSONParser p = new JSONParser(json);
+ return getVal(p);
+ }
+
+ public static Object getVal(JSONParser parser) throws IOException {
+ return new ObjectBuilder(parser).getVal();
+ }
+
+ final JSONParser parser;
+
+ public ObjectBuilder(JSONParser parser) throws IOException {
+ this.parser = parser;
+ if (parser.lastEvent() == 0) parser.nextEvent();
+ }
+
+
+ public Object getVal() throws IOException {
+ int ev = parser.lastEvent();
+ switch (ev) {
+ case JSONParser.STRING:
+ return getString();
+ case JSONParser.LONG:
+ return getLong();
+ case JSONParser.NUMBER:
+ return getNumber();
+ case JSONParser.BIGNUMBER:
+ return getBigNumber();
+ case JSONParser.BOOLEAN:
+ return getBoolean();
+ case JSONParser.NULL:
+ return getNull();
+ case JSONParser.OBJECT_START:
+ return getObject();
+ case JSONParser.OBJECT_END:
+ return null; // OR ERROR?
+ case JSONParser.ARRAY_START:
+ return getArray();
+ case JSONParser.ARRAY_END:
+ return null; // OR ERROR?
+ case JSONParser.EOF:
+ return null; // OR ERROR?
+ default:
+ return null; // OR ERROR?
+ }
+ }
+
+
+ public Object getString() throws IOException {
+ return parser.getString();
+ }
+
+ public Object getLong() throws IOException {
+ return Long.valueOf(parser.getLong());
+ }
+
+ public Object getNumber() throws IOException {
+ CharArr num = parser.getNumberChars();
+ String numstr = num.toString();
+ double d = Double.parseDouble(numstr);
+ if (!Double.isInfinite(d)) return Double.valueOf(d);
+ // TODO: use more efficient constructor in Java5
+ return new BigDecimal(num.buf, num.start, num.size());
+ }
+
+ public Object getBigNumber() throws IOException {
+ CharArr num = parser.getNumberChars();
+ String numstr = num.toString();
+ for (int ch; (ch = num.read()) != -1; ) {
+ if (ch == '.' || ch == 'e' || ch == 'E') return new BigDecimal(numstr);
+ }
+ return new BigInteger(numstr);
+ }
+
+ public Object getBoolean() throws IOException {
+ return parser.getBoolean();
+ }
+
+ public Object getNull() throws IOException {
+ parser.getNull();
+ return null;
+ }
+
+ public Object newObject() throws IOException {
+ return new LinkedHashMap<Object, Object>();
+ }
+
+ public Object getKey() throws IOException {
+ return parser.getString();
+ }
+
+ @SuppressWarnings("unchecked")
+ public void addKeyVal(Object map, Object key, Object val) throws IOException {
+ /* Object prev = */
+ ((Map<Object, Object>) map).put(key, val);
+ // TODO: test for repeated value?
+ }
+
+ public Object objectEnd(Object obj) {
+ return obj;
+ }
+
+
+ public Object getObject() throws IOException {
+ Object m = newObject();
+ for (; ; ) {
+ int ev = parser.nextEvent();
+ if (ev == JSONParser.OBJECT_END) return objectEnd(m);
+ Object key = getKey();
+ ev = parser.nextEvent();
+ Object val = getVal();
+ addKeyVal(m, key, val);
+ }
+ }
+
+ public Object newArray() {
+ return new ArrayList<Object>();
+ }
+
+ @SuppressWarnings("unchecked")
+ public void addArrayVal(Object arr, Object val) throws IOException {
+ ((List<Object>) arr).add(val);
+ }
+
+ public Object endArray(Object arr) {
+ return arr;
+ }
+
+ public Object getArray() throws IOException {
+ Object arr = newArray();
+ for (; ; ) {
+ int ev = parser.nextEvent();
+ if (ev == JSONParser.ARRAY_END) return endArray(arr);
+ Object val = getVal();
+ addArrayVal(arr, val);
+ }
+ }
+
+}
diff --git a/solr/solrj/src/test/org/noggit/TestJSONParser.java b/solr/solrj/src/test/org/noggit/TestJSONParser.java
new file mode 100644
index 0000000..8446cf9
--- /dev/null
+++ b/solr/solrj/src/test/org/noggit/TestJSONParser.java
@@ -0,0 +1,690 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+public class TestJSONParser extends SolrTestCaseJ4 {
+
+ // these are to aid in debugging if an unexpected error occurs
+ static int parserType;
+ static int bufferSize;
+ static String parserInput;
+ static JSONParser lastParser;
+
+ static int flags = JSONParser.FLAGS_DEFAULT; // the default
+
+ public static String lastParser() {
+ return "parserType=" + parserType
+ + (parserType==1 ? " bufferSize=" + bufferSize : "")
+ + " parserInput='" + parserInput + "'" + "flags : " + lastParser.flags;
+ }
+
+ public static JSONParser getParser(String s) {
+ return getParser(s, random().nextInt(2), -1);
+ }
+
+ public static JSONParser getParser(String s, int type, int bufSize) {
+ parserInput = s;
+ parserType = type;
+
+ JSONParser parser=null;
+ switch (type) {
+ case 0:
+ // test directly using input buffer
+ parser = new JSONParser(s.toCharArray(),0,s.length());
+ break;
+ case 1:
+ // test using Reader...
+ // small input buffers can help find bugs on boundary conditions
+
+ if (bufSize < 1) bufSize = random().nextInt(25) + 1;
+ bufferSize = bufSize;// record in case there is an error
+ parser = new JSONParser(new StringReader(s), new char[bufSize]);
+ break;
+ }
+ if (parser == null) return null;
+
+ lastParser = parser;
+
+ if (flags != JSONParser.FLAGS_DEFAULT) {
+ parser.setFlags(flags);
+ }
+
+ return parser;
+ }
+
+ /** for debugging purposes
+ public void testSpecific() throws Exception {
+ JSONParser parser = getParser("[0",1,1);
+ for (;;) {
+ int ev = parser.nextEvent();
+ if (ev == JSONParser.EOF) {
+ break;
+ } else {
+ System.out.println("got " + JSONParser.getEventString(ev));
+ }
+ }
+ }
+ **/
+
+ public static byte[] events = new byte[256];
+ static {
+ events['{'] = JSONParser.OBJECT_START;
+ events['}'] = JSONParser.OBJECT_END;
+ events['['] = JSONParser.ARRAY_START;
+ events[']'] = JSONParser.ARRAY_END;
+ events['s'] = JSONParser.STRING;
+ events['b'] = JSONParser.BOOLEAN;
+ events['l'] = JSONParser.LONG;
+ events['n'] = JSONParser.NUMBER;
+ events['N'] = JSONParser.BIGNUMBER;
+ events['0'] = JSONParser.NULL;
+ events['e'] = JSONParser.EOF;
+ }
+
+ // match parser states with the expected states
+ public static void parse(JSONParser p, String input, String expected) throws IOException {
+ expected += "e";
+ for (int i=0; i<expected.length(); i++) {
+ int ev = p.nextEvent();
+ int expect = events[expected.charAt(i)];
+ if (ev != expect) {
+ fail("Expected " + expect + ", got " + ev
+ + "\n\tINPUT=" + input
+ + "\n\tEXPECTED=" + expected
+ + "\n\tAT=" + i + " (" + expected.charAt(i) + ")");
+ }
+ }
+ }
+
+ public static void parse(String input, String expected) throws IOException {
+ String in = input;
+ if ((flags & JSONParser.ALLOW_SINGLE_QUOTES)==0 || random().nextBoolean()) {
+ in = in.replace('\'', '"');
+ }
+
+ for (int i=0; i<Integer.MAX_VALUE; i++) {
+ JSONParser p = getParser(in,i,-1);
+ if (p==null) break;
+ parse(p,in,expected);
+ }
+
+ testCorruption(input, 100000);
+
+ }
+
+ public static void testCorruption(String input, int iter) {
+ char[] arr = new char[input.length()];
+
+ for (int i=0; i<iter; i++) {
+ input.getChars(0, arr.length, arr, 0);
+ int changes = random().nextInt(arr.length>>1) + 1;
+ for (int j=0; j<changes; j++) {
+ char ch;
+ switch (random().nextInt(31)) {
+ case 0: ch = 0; break;
+ case 1: ch = '['; break;
+ case 2: ch = ']'; break;
+ case 3: ch = '{'; break;
+ case 4: ch = '}'; break;
+ case 5: ch = '"'; break;
+ case 6: ch = '\''; break;
+ case 7: ch = ' '; break;
+ case 8: ch = '\r'; break;
+ case 9: ch = '\n'; break;
+ case 10:ch = '\t'; break;
+ case 11:ch = ','; break;
+ case 12:ch = ':'; break;
+ case 13:ch = '.'; break;
+ case 14:ch = 'a'; break;
+ case 15:ch = 'e'; break;
+ case 16:ch = '0'; break;
+ case 17:ch = '1'; break;
+ case 18:ch = '+'; break;
+ case 19:ch = '-'; break;
+ case 20:ch = 't'; break;
+ case 21:ch = 'f'; break;
+ case 22:ch = 'n'; break;
+ case 23:ch = '/'; break;
+ case 24:ch = '\\'; break;
+ case 25:ch = 'u'; break;
+ case 26:ch = '\u00a0'; break;
+ default:ch = (char) random().nextInt(256);
+ }
+
+ arr[random().nextInt(arr.length)] = ch;
+ }
+
+
+ JSONParser parser = getParser(new String(arr));
+ parser.setFlags( random().nextInt() ); // set random parser flags
+
+ int ret = 0;
+ try {
+ for (;;) {
+ int ev = parser.nextEvent();
+ if (random().nextBoolean()) {
+ // see if we can read the event
+ switch (ev) {
+ case JSONParser.STRING: ret += parser.getString().length(); break;
+ case JSONParser.BOOLEAN: ret += parser.getBoolean() ? 1 : 2; break;
+ case JSONParser.BIGNUMBER: ret += parser.getNumberChars().length(); break;
+ case JSONParser.NUMBER: ret += parser.getDouble(); break;
+ case JSONParser.LONG: ret += parser.getLong(); break;
+ default: ret += ev;
+ }
+ }
+
+ if (ev == JSONParser.EOF) break;
+ }
+ } catch (IOException ex) {
+ // shouldn't happen
+ System.out.println(ret); // use ret
+ } catch (JSONParser.ParseException ex) {
+ // OK
+ } catch (Throwable ex) {
+ ex.printStackTrace();
+ System.out.println(lastParser());
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+
+
+ public static class Num {
+ public String digits;
+ public Num(String digits) {
+ this.digits = digits;
+ }
+ @Override
+ public String toString() { return new String("NUMBERSTRING("+digits+")"); }
+ @Override
+ public boolean equals(Object o) {
+ return (getClass() == o.getClass() && digits.equals(((Num) o).digits));
+ }
+ @Override
+ public int hashCode() {
+ return digits.hashCode();
+ }
+ }
+
+ public static class BigNum extends Num {
+ @Override
+ public String toString() { return new String("BIGNUM("+digits+")"); }
+ public BigNum(String digits) { super(digits); }
+ }
+
+ // Oh, what I wouldn't give for Java5 varargs and autoboxing
+ public static Long o(int l) { return (long) l; }
+ public static Long o(long l) { return l; }
+ public static Double o(double d) { return d; }
+ public static Boolean o(boolean b) { return b; }
+ public static Num n(String digits) { return new Num(digits); }
+ public static Num bn(String digits) { return new BigNum(digits); }
+ public static Object t = Boolean.TRUE;
+ public static Object f = Boolean.FALSE;
+ public static Object a = new Object(){@Override
+ public String toString() {return "ARRAY_START";}};
+ public static Object A = new Object(){@Override
+ public String toString() {return "ARRAY_END";}};
+ public static Object m = new Object(){@Override
+ public String toString() {return "OBJECT_START";}};
+ public static Object M = new Object(){@Override
+ public String toString() {return "OBJECT_END";}};
+ public static Object N = new Object(){@Override
+ public String toString() {return "NULL";}};
+ public static Object e = new Object(){@Override
+ public String toString() {return "EOF";}};
+
+ // match parser states with the expected states
+ public static void parse(JSONParser p, String input, Object[] expected) throws IOException {
+ for (int i=0; i<expected.length; i++) {
+ int ev = p.nextEvent();
+ Object exp = expected[i];
+ Object got = null;
+
+ switch(ev) {
+ case JSONParser.ARRAY_START: got=a; break;
+ case JSONParser.ARRAY_END: got=A; break;
+ case JSONParser.OBJECT_START: got=m; break;
+ case JSONParser.OBJECT_END: got=M; break;
+ case JSONParser.LONG: got=o(p.getLong()); break;
+ case JSONParser.NUMBER:
+ if (exp instanceof Double) {
+ got = o(p.getDouble());
+ } else {
+ got = n(p.getNumberChars().toString());
+ }
+ break;
+ case JSONParser.BIGNUMBER: got=bn(p.getNumberChars().toString()); break;
+ case JSONParser.NULL: got=N; p.getNull(); break; // optional
+ case JSONParser.BOOLEAN: got=o(p.getBoolean()); break;
+ case JSONParser.EOF: got=e; break;
+ case JSONParser.STRING: got=p.getString(); break;
+ default: got="Unexpected Event Number " + ev;
+ }
+
+ if (!(exp==got || exp.equals(got))) {
+ fail("Fail: String='" + input + "'" + "\n\tINPUT=" + got + "\n\tEXPECTED=" + exp + "\n\tAT RULE " + i);
+ }
+ }
+ }
+
+
+ public static void parse(String input, Object[] expected) throws IOException {
+ parse(input, (flags & JSONParser.ALLOW_SINGLE_QUOTES)==0 || random().nextBoolean(), expected);
+ }
+
+ public static void parse(String input, boolean changeSingleQuote, Object[] expected) throws IOException {
+ String in = input;
+ if (changeSingleQuote) {
+ in = in.replace('\'', '"');
+ }
+ for (int i=0; i<Integer.MAX_VALUE; i++) {
+ JSONParser p = getParser(in,i,-1);
+ if (p == null) break;
+ parse(p,in,expected);
+ }
+ }
+
+
+ public static void err(String input) throws IOException {
+ try {
+ JSONParser p = getParser(input);
+ while (p.nextEvent() != JSONParser.EOF) {}
+ } catch (Exception e) {
+ return;
+ }
+ fail("Input should failed:'" + input + "'");
+ }
+
+ @Test
+ public void testNull() throws IOException {
+ flags = JSONParser.FLAGS_STRICT;
+ err("nul");
+ err("n");
+ err("nullz");
+ err("[nullz]");
+ flags = JSONParser.FLAGS_DEFAULT;
+
+ parse("[null]","[0]");
+ parse("{'hi':null}",new Object[]{m,"hi",N,M,e});
+ }
+
+ @Test
+ public void testBool() throws IOException {
+ flags = JSONParser.FLAGS_STRICT;
+ err("[True]");
+ err("[False]");
+ err("[TRUE]");
+ err("[FALSE]");
+ err("[truex]");
+ err("[falsex]");
+ err("[tru]");
+ err("[fals]");
+ err("[tru");
+ err("[fals");
+ err("t");
+ err("f");
+ flags = JSONParser.FLAGS_DEFAULT;
+
+ parse("[false,true, false , true ]",new Object[]{a,f,t,f,t,A,e});
+ }
+
+ @Test
+ public void testString() throws IOException {
+ // NOTE: single quotes are converted to double quotes by this
+ // testsuite!
+ err("[']");
+ err("[',]");
+ err("{'}");
+ err("{',}");
+
+ err("['\\u111']");
+ err("['\\u11']");
+ err("['\\u1']");
+ err("['\\']");
+
+
+ flags = JSONParser.FLAGS_STRICT;
+ err("['\\ ']"); // escape of non-special char
+ err("['\\U1111']"); // escape of non-special char
+ flags = JSONParser.FLAGS_DEFAULT;
+
+ parse("['\\ ']", new Object[]{a, " ", A, e}); // escape of non-special char
+ parse("['\\U1111']", new Object[]{a, "U1111", A, e}); // escape of non-special char
+
+
+
+ parse("['']",new Object[]{a,"",A,e});
+ parse("['\\\\']",new Object[]{a,"\\",A,e});
+ parse("['X\\\\']",new Object[]{a,"X\\",A,e});
+ parse("['\\\\X']",new Object[]{a,"\\X",A,e});
+ parse("[\"\\\"\"]",new Object[]{a,"\"",A,e});
+
+ parse("['\\'']", true, new Object[]{a,"\"",A,e});
+ parse("['\\'']", false, new Object[]{a,"'",A,e});
+
+
+ String esc="\\n\\r\\tX\\b\\f\\/\\\\X\\\"";
+ String exp="\n\r\tX\b\f/\\X\"";
+ parse("['" + esc + "']",new Object[]{a,exp,A,e});
+ parse("['" + esc+esc+esc+esc+esc + "']",new Object[]{a,exp+exp+exp+exp+exp,A,e});
+
+ esc="\\u004A";
+ exp="\u004A";
+ parse("['" + esc + "']",new Object[]{a,exp,A,e});
+
+ esc="\\u0000\\u1111\\u2222\\u12AF\\u12BC\\u19DE";
+ exp="\u0000\u1111\u2222\u12AF\u12BC\u19DE";
+ parse("['" + esc + "']",new Object[]{a,exp,A,e});
+
+ }
+
+ @Test
+ public void testNumbers() throws IOException {
+ flags = JSONParser.FLAGS_STRICT;
+
+ err("[00]");
+ err("[003]");
+ err("[00.3]");
+ err("[1e1.1]");
+ err("[+1]");
+ err("[NaN]");
+ err("[Infinity]");
+ err("[--1]");
+
+ flags = JSONParser.FLAGS_DEFAULT;
+
+ String lmin = "-9223372036854775808";
+ String lminNot = "-9223372036854775809";
+ String lmax = "9223372036854775807";
+ String lmaxNot = "9223372036854775808";
+
+ String bignum="12345678987654321357975312468642099775533112244668800152637485960987654321";
+
+ parse("[0,1,-1,543,-876]", new Object[]{a,o(0),o(1),o(-1),o(543),o(-876),A,e});
+ parse("[-0]",new Object[]{a,o(0),A,e});
+
+
+ parse("["+lmin +"," + lmax+"]",
+ new Object[]{a,o(Long.MIN_VALUE),o(Long.MAX_VALUE),A,e});
+
+ parse("["+bignum+"]", new Object[]{a,bn(bignum),A,e});
+ parse("["+"-"+bignum+"]", new Object[]{a,bn("-"+bignum),A,e});
+
+ parse("["+lminNot+"]",new Object[]{a,bn(lminNot),A,e});
+ parse("["+lmaxNot+"]",new Object[]{a,bn(lmaxNot),A,e});
+
+ parse("["+lminNot + "," + lmaxNot + "]",
+ new Object[]{a,bn(lminNot),bn(lmaxNot),A,e});
+
+ // bignum many digits on either side of decimal
+ String t = bignum + "." + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ err("[" + t+".1" + "]"); // extra decimal
+ err("[" + "-"+t+".1" + "]");
+
+ // bignum exponent w/o fraction
+ t = "1" + "e+" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ t = "1" + "E+" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ t = "1" + "e" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ t = "1" + "E" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ t = "1" + "e-" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ t = "1" + "E-" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+ t = bignum + "e+" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ t = bignum + "E-" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ t = bignum + "e" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+ t = bignum + "." + bignum + "e" + bignum;
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+ err("[1E]");
+ err("[1E-]");
+ err("[1E+]");
+ err("[1E+.3]");
+ err("[1E+0.3]");
+ err("[1E+1e+3]");
+ err("["+bignum+"e"+"]");
+ err("["+bignum+"e-"+"]");
+ err("["+bignum+"e+"+"]");
+ err("["+bignum+"."+bignum+"."+bignum+"]");
+
+
+ double[] vals = new double[] {0,0.1,1.1,
+ Double.MAX_VALUE,
+ Double.MIN_VALUE,
+ 2.2250738585072014E-308, /* Double.MIN_NORMAL */
+ };
+ for (int i=0; i<vals.length; i++) {
+ double d = vals[i];
+ parse("["+d+","+-d+"]", new Object[]{a,o(d),o(-d),A,e});
+ }
+
+ // MIN_NORMAL has the max number of digits (23), so check that
+ // adding an extra digit causes BIGNUM to be returned.
+ t = "2.2250738585072014E-308" + "0";
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+ // check it works with a leading zero too
+ t = "0.2250738585072014E-308" + "0";
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+ // check that overflow detection is working properly w/ numbers that don't cause a wrap to negatives
+ // when multiplied by 10
+ t = "1910151821265210155" + "0";
+ parse("["+t+","+"-"+t+"]", new Object[]{a,bn(t),bn("-"+t),A,e});
+
+ for (int i=0; i<1000000; i++) {
+ long val = random().nextLong();
+ String sval = Long.toString(val);
+ JSONParser parser = getParser("["+val+"]");
+ parser.nextEvent();
+ assertTrue(parser.nextEvent() == JSONParser.LONG);
+ if (random().nextBoolean()) {
+ assertEquals(val, parser.getLong());
+ } else {
+ CharArr chars = parser.getNumberChars();
+ assertEquals(sval, chars.toString());
+ }
+ }
+
+ }
+
+ @Test
+ public void testArray() throws IOException {
+ parse("[]","[]");
+ parse("[ ]","[]");
+ parse(" \r\n\t[\r\t\n ]\r\n\t ","[]");
+
+ parse("[0]","[l]");
+ parse("['0']","[s]");
+ parse("[0,'0',0.1]","[lsn]");
+
+ parse("[[[[[]]]]]","[[[[[]]]]]");
+ parse("[[[[[0]]]]]","[[[[[l]]]]]");
+
+ err("]");
+ err("[");
+ err("[[]");
+ err("[]]");
+ err("[}");
+ err("{]");
+ err("['a':'b']");
+
+ flags=JSONParser.FLAGS_STRICT;
+ err("[,]"); // test that extra commas fail
+ err("[[],]");
+ err("['a',]");
+ err("['a',]");
+ flags=JSONParser.FLAGS_DEFAULT;
+
+ parse("[,]","[]"); // test extra commas
+ parse("[,,]","[]");
+ parse("[,,,]","[]");
+ parse("[[],]","[[]]");
+ parse("[[,],]","[[]]");
+ parse("[[,,],,]","[[]]");
+ parse("[,[,,],,]","[[]]");
+ parse("[,5,[,,5],,]","[l[l]]");
+
+ }
+
+ @Test
+ public void testObject() throws IOException {
+ parse("{}","{}");
+ parse("{}","{}");
+ parse(" \r\n\t{\r\t\n }\r\n\t ","{}");
+
+ parse("{'':null}","{s0}");
+
+ err("}");
+ err("[}]");
+ err("{");
+ err("[{]");
+ err("{{}");
+ err("[{{}]");
+ err("{}}");
+ err("[{}}]");
+ err("{1}");
+ err("[{1}]");
+ err("{'a'}");
+ err("{'a','b'}");
+ err("{[]:'b'}");
+ err("{{'a':'b'}:'c'}");
+ err("{'a','b'}}");
+
+ // bare strings allow these to pass
+ flags=JSONParser.FLAGS_STRICT;
+ err("{null:'b'}");
+ err("{true:'b'}");
+ err("{false:'b'}");
+ err("{,}"); // test that extra commas fail
+ err("{{},}");
+ err("{'a':'b',}");
+ flags=JSONParser.FLAGS_DEFAULT;
+
+ parse("{}", new Object[]{m,M,e});
+ parse("{,}", new Object[]{m,M,e});
+ parse("{,,}", new Object[]{m,M,e});
+ parse("{'a':{},}", new Object[]{m,"a",m,M,M,e});
+ parse("{'a':{},,}", new Object[]{m,"a",m,M,M,e});
+ parse("{,'a':{,},,}", new Object[]{m,"a",m,M,M,e});
+ parse("{'a':'b'}", new Object[]{m,"a","b",M,e});
+ parse("{'a':5}", new Object[]{m,"a",o(5),M,e});
+ parse("{'a':null}", new Object[]{m,"a",N,M,e});
+ parse("{'a':[]}", new Object[]{m,"a",a,A,M,e});
+ parse("{'a':{'b':'c'}}", new Object[]{m,"a",m,"b","c",M,M,e});
+
+ String big = "Now is the time for all good men to come to the aid of their country!";
+ String bigger = big+big+big+big+big;
+ parse("{'"+bigger+"':'"+bigger+"','a':'b'}", new Object[]{m,bigger,bigger,"a","b",M,e});
+
+
+ flags=JSONParser.ALLOW_UNQUOTED_KEYS;
+ parse("{a:'b'}", new Object[]{m,"a","b",M,e});
+ parse("{null:'b'}", new Object[]{m,"null","b",M,e});
+ parse("{true: 'b'}", new Object[]{m,"true","b",M,e});
+ parse("{ false :'b'}", new Object[]{m,"false","b",M,e});
+ parse("{null:null, true : true , false : false , x:'y',a:'b'}", new Object[]{m,"null",N,"true",t,"false",f,"x","y","a","b",M,e});
+ flags = JSONParser.FLAGS_DEFAULT | JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT;
+ parse("{'a'{'b':'c'}}", new Object[]{m, "a", m, "b", "c", M, M, e});
+ parse("{'a': [{'b':'c'} {'b':'c'}]}", new Object[]{m, "a",a, m, "b", "c", M, m, "b", "c", M,A, M, e});
+ parse("{'a' [{'b':'c'} {'b':'c'}]}", new Object[]{m, "a", a, m, "b", "c", M, m, "b", "c", M, A, M, e});
+ parse("{'a':[['b']['c']]}", new Object[]{m, "a", a, a, "b", A, a, "c", A, A, M, e});
+ parse("{'a': {'b':'c'} 'd': {'e':'f'}}", new Object[]{m, "a", m, "b", "c",M, "d", m,"e","f", M, M, e});
+ parse("{'a': {'b':'c'} d: {'e':'f'}}", new Object[]{m, "a", m, "b", "c",M, "d", m,"e","f", M, M, e});
+
+ flags = JSONParser.FLAGS_DEFAULT | JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT | JSONParser.OPTIONAL_OUTER_BRACES;
+ parse("'a':{'b':'c'}", new Object[]{m, "a", m, "b", "c", M, M, e});
+ parse("'a':{'b':'c'}", true, new Object[]{m, "a", m, "b", "c", M, M, e});
+ parse("a:'b'", new Object[]{m, "a", "b", M, e});
+
+
+ flags = JSONParser.FLAGS_DEFAULT;
+
+ }
+
+ @Test
+ public void testBareString() throws Exception {
+ flags=JSONParser.ALLOW_UNQUOTED_STRING_VALUES | JSONParser.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER;
+ String[] strings = new String[] {"t","f","n","a","tru","fals","nul","abc","trueX","falseXY","nullZ","truetrue", "$true", "a.bc.def","a_b-c/d"};
+
+ for (String s : strings) {
+ parse(s, new Object[]{s, e});
+ parse("[" + s + "]", new Object[]{a, s, A, e});
+ parse("[ " + s + ", "+s +" ]", new Object[]{a, s, s, A, e});
+ parse("[" + s + ","+s +"]", new Object[]{a, s, s, A, e});
+ parse("\u00a0[\u00a0\r\n\t\u00a0" + s + "\u00a0,\u00a0\u00a0"+s +"\u00a0]\u00a0", new Object[]{a, s, s, A, e});
+ }
+
+ flags |= JSONParser.ALLOW_UNQUOTED_KEYS;
+ for (String s : strings) {
+ parse("{" + s + ":" + s + "}", new Object[]{m, s, s, M, e});
+ parse("{ " + s + " \t\n\r:\t\n\r " + s + "\t\n\r}", new Object[]{m, s, s, M, e});
+ }
+
+ parse("{true:true, false:false, null:null}",new Object[]{m,"true",t,"false",f,"null",N,M,e});
+
+ flags=JSONParser.FLAGS_DEFAULT;
+ }
+
+
+ @Test
+ public void testAPI() throws IOException {
+ JSONParser p = new JSONParser("[1,2]");
+ assertEquals(JSONParser.ARRAY_START, p.nextEvent());
+ // no nextEvent for the next objects...
+ assertEquals(1,p.getLong());
+ assertEquals(2,p.getLong());
+ }
+
+
+ @Test
+ public void testComments() throws IOException {
+ parse("#pre comment\n{//before a\n 'a' /* after a **/ #before separator\n : /* after separator */ {/*before b*/'b'#after b\n://before c\n'c'/*after c*/}/*after close*/}#post comment no EOL", new Object[]{m,"a",m,"b","c",M,M,e});
+ }
+
+ // rfc7159 permits any JSON values to be top level values
+ @Test
+ public void testTopLevelValues() throws Exception {
+ parse("\"\"", new Object[]{""});
+ parse("\"hello\"", new Object[]{"hello"});
+ parse("true", new Object[]{t});
+ parse("false", new Object[]{f});
+ parse("null", new Object[]{N});
+ parse("42", new Object[]{42L});
+ parse("1.414", new Object[]{1.414d});
+ parse("/*comment*/1.414/*more comment*/", new Object[]{1.414d});
+ }
+}
diff --git a/solr/solrj/src/test/org/noggit/TestJSONWriter.java b/solr/solrj/src/test/org/noggit/TestJSONWriter.java
new file mode 100644
index 0000000..d9a4f34
--- /dev/null
+++ b/solr/solrj/src/test/org/noggit/TestJSONWriter.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+public class TestJSONWriter extends SolrTestCaseJ4 {
+
+ // note - TestObjectBuilder also exercises JSONWriter
+
+ public void test(String expected, Object val, int indent) throws IOException {
+ expected = expected.replace('\'','"');
+ String s1 = JSONUtil.toJSON(val, indent);
+ assertEquals(s1, expected);
+ }
+
+ public static List<Object> L(Object... lst) {
+ return Arrays.asList(lst);
+ }
+ public static Object[] A(Object... lst) {
+ return lst;
+ }
+ public static Map<String,Object> O(Object... lst) {
+ LinkedHashMap<String,Object> map = new LinkedHashMap<String,Object>();
+ for (int i=0; i<lst.length; i+=2) {
+ map.put(lst[i].toString(), lst[i+1]);
+ }
+ return map;
+ }
+
+ // NOTE: the specifics of indentation may change in the future!
+ @Test
+ public void testWriter() throws Exception {
+ test("[]",L(),2);
+ test("{}",O(),2);
+ test("[\n 10,\n 20]", L(10,20), 2);
+ test("{\n 'a':10,\n 'b':{\n 'c':20,\n 'd':30}}", O("a",10,"b",O("c",20,"d",30)), 1);
+
+ test("['\\r\\n\\u0000\\'']", L("\r\n\u0000\""),2);
+
+ }
+
+
+ public static class Unknown {
+ @Override
+ public String toString() {
+ return "a,\"b\",c";
+ }
+ }
+
+ public static class Custom implements JSONWriter.Writable {
+ @Override
+ public void write(JSONWriter writer) {
+ Map<String,Integer> val = new LinkedHashMap<String,Integer>();
+ val.put("a",1);
+ val.put("b",2);
+ writer.write(val);
+ }
+ }
+
+ @Test
+ public void testWritable() throws Exception {
+ test("[{'a':1,'b':2}]", L(new Custom()), -1);
+ test("[10,{'a':1,'b':2},20]", L(10, new Custom(), 20), -1);
+ }
+
+ @Test
+ public void testUnknown() throws Exception {
+ test("['a,\\\"b\\\",c']", L(new Unknown()), -1);
+ }
+
+}
diff --git a/solr/solrj/src/test/org/noggit/TestObjectBuilder.java b/solr/solrj/src/test/org/noggit/TestObjectBuilder.java
new file mode 100644
index 0000000..e4a7504
--- /dev/null
+++ b/solr/solrj/src/test/org/noggit/TestObjectBuilder.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.noggit;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+public class TestObjectBuilder extends SolrTestCaseJ4 {
+
+ public void _test(String val, Object expected) throws IOException {
+ val = val.replace('\'','"');
+ Object v = ObjectBuilder.fromJSON(val);
+
+ String s1 = JSONUtil.toJSON(v,-1);
+ String s2 = JSONUtil.toJSON(expected,-1);
+ assertEquals(s1, s2);
+
+ // not make sure that it round-trips correctly
+ JSONParser p2 = TestJSONParser.getParser(s1);
+ Object v2 = ObjectBuilder.getVal(p2);
+ String s3 = JSONUtil.toJSON(v2,-1);
+ assertEquals(s1, s3);
+ }
+
+ public static List<Object> L(Object... lst) {
+ return Arrays.asList(lst);
+ }
+ public static Object[] A(Object... lst) {
+ return lst;
+ }
+ public static Map<String, Object> O(Object... lst) {
+ LinkedHashMap<String,Object> map = new LinkedHashMap<String,Object>();
+ for (int i=0; i<lst.length; i+=2) {
+ map.put(lst[i].toString(), lst[i+1]);
+ }
+ return map;
+ }
+ @Test
+ public void testWithoutStartingBraces() throws IOException {
+ JSONParser parser = new JSONParser("a: {key:val1}b:{key:val2}");
+ parser.setFlags(JSONParser.FLAGS_DEFAULT | JSONParser.OPTIONAL_OUTER_BRACES| JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT);
+ ObjectBuilder objectBuilder = new ObjectBuilder(parser);
+ String s1 = JSONUtil.toJSON(objectBuilder.getObject(),-1);
+ String expected = JSONUtil.toJSON(O("a", O("key", "val1"), "b", O("key", "val2")),-1);
+ assertEquals(s1, expected);
+ }
+
+ public void _testVariations(String str, Object expected) throws IOException {
+ _test("["+str+"]", L(expected));
+ _test("["+str+","+str+"]", L(expected, expected));
+ _test("["+str+","+str+"]", A(expected, expected));
+ _test("{'foo':"+str+"}", O("foo",expected));
+ _test("{'foo':"+str+",'bar':{'a':"+str+"},'baz':["+str+"],'zzz':["+str+"]}",
+ O("foo",expected,"bar",O("a",expected),"baz", L(expected), "zzz", A(expected)));
+
+ }
+ @Test
+ public void testBuilder() throws IOException {
+ _testVariations("[]", L());
+ _testVariations("[]", L());
+ _testVariations("{}", O());
+ _testVariations("[[]]", L(L()));
+ _testVariations("{'foo':{}}", O("foo",O()));
+ _testVariations("[false,true,1,1.4,null,'hi']", L(false, true, 1, 1.4, null, "hi"));
+ _testVariations("'hello'", "hello".toCharArray());
+
+ // test array types
+ _testVariations("[[10,20],['a','b']]", L(A(10,20),A("a","b")));
+ _testVariations("[1,2,3]", new int[]{1,2,3});
+ _testVariations("[1,2,3]", new long[]{1,2,3});
+ _testVariations("[1.0,2.0,3.0]", new float[]{1,2,3});
+ _testVariations("[1.0,2.0,3.0]", new double[]{1,2,3});
+ _testVariations("[1,2,3]", new short[]{1,2,3});
+ _testVariations("[1,2,3]", new byte[]{1,2,3});
+ _testVariations("[false,true,false]", new boolean[]{false,true,false});
+ }
+
+}
\ No newline at end of file