You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2016/12/13 20:30:09 UTC
[02/42] lucene-solr:feature/metrics: SOLR-9768 RecordingJsonParser
produces incomplete json (Wojciech Stryszyk via ab)
SOLR-9768 RecordingJsonParser produces incomplete json (Wojciech Stryszyk via ab)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/590d31f3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/590d31f3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/590d31f3
Branch: refs/heads/feature/metrics
Commit: 590d31f311c092aa97bc64b1a28a9dbf934b0e52
Parents: 02c6877
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Nov 29 21:11:40 2016 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Nov 29 21:12:49 2016 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../apache/solr/util/RecordingJSONParser.java | 17 ++++++++--
.../solr/common/util/TestJsonRecordReader.java | 35 ++++++++++++++++----
3 files changed, 45 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590d31f3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index cead9af..448f2d7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -216,6 +216,8 @@ Bug Fixes
* SOLR-5260: Facet search on a docvalue field in a multi shard collection (Trym M�ller, Erick Erickson)
+* SOLR-9768: RecordingJsonParser produces incomplete json (Wojciech Stryszyk via ab)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590d31f3/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
index 030913d..a85610b 100644
--- a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
+++ b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
@@ -29,7 +29,9 @@ public class RecordingJSONParser extends JSONParser {
private StringBuilder sb = new StringBuilder();
private boolean objectStarted = false;
- public long lastMarkedPosition = 0;
+ private long lastMarkedPosition = 0;
+ private long lastGlobalPosition = 0;
+ private static final int BUFFER_SIZE = 8192;
public RecordingJSONParser(Reader in) {
@@ -39,7 +41,7 @@ public class RecordingJSONParser extends JSONParser {
}
static char[] getChars() {
- buf.set(new char[8192]);
+ buf.set(new char[BUFFER_SIZE]);
return buf.get();
}
@@ -68,11 +70,22 @@ public class RecordingJSONParser extends JSONParser {
if(currPosition < 0){
System.out.println("ERROR");
}
+
if (currPosition > lastMarkedPosition) {
for (long i = lastMarkedPosition; i < currPosition; i++) {
recordChar(bufCopy[(int) i]);
}
+ } else if (currPosition < lastMarkedPosition) {
+ for (long i = 0; i < currPosition; i++) {
+ recordChar(bufCopy[(int) i]);
+ }
+ } else if (currPosition == BUFFER_SIZE && lastGlobalPosition != globalPosition) {
+ for (long i = 0; i < currPosition; i++) {
+ recordChar(bufCopy[(int) i]);
+ }
}
+
+ lastGlobalPosition = globalPosition;
lastMarkedPosition = currPosition;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/590d31f3/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
index d59dea3..da75a43 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
@@ -16,11 +16,6 @@
*/
package org.apache.solr.common.util;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.util.RecordingJSONParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
@@ -31,6 +26,12 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.lang.StringUtils;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.util.RecordingJSONParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class TestJsonRecordReader extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -129,18 +130,32 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
" \"nested_inside\" : \"check check check 1\"\n" +
" }\n" +
"}";
+
String json2 =
" {\n" +
" \"id\" : \"345\",\n" +
+ " \"payload\": \""+ StringUtils.repeat("0123456789", 819) +
+ "\",\n" +
" \"description\": \"Testing /json/docs srcField 2\",\n" +
"\n" +
" \"nested_data\" : {\n" +
" \"nested_inside\" : \"check check check 2\"\n" +
" }\n" +
"}";
- JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id"));
- RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2));
+ String json3 =
+ " {\n" +
+ " \"id\" : \"678\",\n" +
+ " \"description\": \"Testing /json/docs srcField 3\",\n" +
+ "\n" +
+ " \"nested_data\" : {\n" +
+ " \"nested_inside\" : \"check check check 3\"\n" +
+ " }\n" +
+ "}";
+
+
+ JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id"));
+ RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2 + json3));
streamer.streamRecords(parser, new JsonRecordReader.Handler() {
int count = 0;
@@ -162,6 +177,12 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
assertEquals(m.get("description"), "Testing /json/docs srcField 2");
assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 2");
}
+ if (count++ == 3) {
+ assertEquals(m.get("id"), "678");
+ assertEquals(m.get("description"), "Testing /json/docs srcField 3");
+ assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 3");
+ }
+
}
});