You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/06/21 13:22:25 UTC
[1/2] lucene-solr:branch_6x: SOLR-9234: srcField parameter works only
when all fields are captured in the /update/json/docs endpoint
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 4a71e03a3 -> 991090755
SOLR-9234: srcField parameter works only when all fields are captured in the /update/json/docs endpoint
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/73d5f1c5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/73d5f1c5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/73d5f1c5
Branch: refs/heads/branch_6x
Commit: 73d5f1c52cd7a9531f07aea6c9f88d1ff253ac64
Parents: 3a9019e
Author: Noble Paul <no...@apache.org>
Authored: Tue Jun 21 18:50:49 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jun 21 18:51:45 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../apache/solr/util/RecordingJSONParser.java | 79 ++++++++++++--------
.../solr/common/util/TestJsonRecordReader.java | 52 +++++++++++++
3 files changed, 103 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/73d5f1c5/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a2c46d2..e950f10 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -47,6 +47,9 @@ Bug Fixes
* SOLR-9161: Change SolrPluginUtils.invokeSetters implementation to accommodate setter variants.
(Christine Poerschke, Steve Rowe, Uwe Schindler)
+* SOLR-9234: srcField parameter works only when all fields are captured in the /update/json/docs
+ endpoint (noble)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/73d5f1c5/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
index 02ff03c..c2ec2ba 100644
--- a/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
+++ b/solr/core/src/java/org/apache/solr/util/RecordingJSONParser.java
@@ -22,56 +22,73 @@ import java.io.Reader;
import org.noggit.CharArr;
import org.noggit.JSONParser;
-public class RecordingJSONParser extends JSONParser{
- public RecordingJSONParser(Reader in) { super(in); }
+public class RecordingJSONParser extends JSONParser {
+ static ThreadLocal<char[]> buf = new ThreadLocal<>();
+ private final char[] bufCopy;
+ //global position is the global position at the beginning of my buffer
+ private long globalPosition = 0;
- private StringBuilder sb = new StringBuilder() ;
- private long position;
- private boolean objectStarted =false;
+ private StringBuilder sb = new StringBuilder();
+ private boolean objectStarted = false;
+ public long lastMarkedPosition = 0;
+ public RecordingJSONParser(Reader in) {
+ super(in, getChars());
+ bufCopy = buf.get();
+ buf.remove();
+ }
- @Override
- protected int getChar() throws IOException {
- int aChar = super.getChar();
- if(aChar == '{') objectStarted =true;
- if(getPosition() >position) recordChar((char) aChar); // check before adding if a pushback happened ignore
- position= getPosition();
- return aChar;
+ static char[] getChars() {
+ buf.set(new char[8192]);
+ return buf.get();
}
private void recordChar(int aChar) {
- if(objectStarted)
+ if (objectStarted) {
sb.append((char) aChar);
+ } else if (aChar == '{') {
+ sb.append((char) aChar);
+ objectStarted = true;
+ }
}
- private void recordStr(String s) {
- if(objectStarted) sb.append(s);
+
+ public void resetBuf() {
+ sb = new StringBuilder();
+ objectStarted = false;
}
@Override
- public CharArr getStringChars() throws IOException {
- CharArr chars = super.getStringChars();
- recordStr(chars.toString());
- position = getPosition();
- // if reading a String , the getStringChars do not return the closing single quote or double quote
- //so, try to capture that
- if(chars.getArray().length >chars.getStart()+chars.size()) {
- char next = chars.getArray()[chars.getStart() + chars.size()];
- if(next =='"' || next == '\'') {
- recordChar(next);
+ public int nextEvent() throws IOException {
+ captureMissing();
+ return super.nextEvent();
+ }
+
+ private void captureMissing() {
+ long currPosition = getPosition() - globalPosition;
+ if(currPosition < 0){
+ System.out.println("ERROR");
+ }
+ if (currPosition > lastMarkedPosition) {
+ for (long i = lastMarkedPosition; i < currPosition; i++) {
+ recordChar(bufCopy[(int) i]);
}
}
- return chars;
+ lastMarkedPosition = currPosition;
}
- public void resetBuf(){
- sb = new StringBuilder();
- objectStarted=false;
- }
+ @Override
+ protected void fill() throws IOException {
+ captureMissing();
+ super.fill();
+ this.globalPosition = getPosition();
+
+ }
public String getBuf() {
- if(sb != null) return sb.toString();
+ captureMissing();
+ if (sb != null) return sb.toString();
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/73d5f1c5/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
index b94808c..d59dea3 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
@@ -18,9 +18,12 @@ package org.apache.solr.common.util;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.RecordingJSONParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
+import java.lang.invoke.MethodHandles;
import java.lang.ref.WeakReference;
import java.util.Arrays;
import java.util.Collections;
@@ -30,6 +33,8 @@ import java.util.concurrent.atomic.AtomicReference;
public class TestJsonRecordReader extends SolrTestCaseJ4 {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
public void testOneLevelSplit() throws IOException {
String json = "{\n" +
" \"a\":\"A\" ,\n" +
@@ -115,6 +120,53 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
}
+ public void testSrcField() throws Exception {
+ String json = "{\n" +
+ " \"id\" : \"123\",\n" +
+ " \"description\": \"Testing /json/docs srcField 1\",\n" +
+ "\n" +
+ " \"nested_data\" : {\n" +
+ " \"nested_inside\" : \"check check check 1\"\n" +
+ " }\n" +
+ "}";
+ String json2 =
+ " {\n" +
+ " \"id\" : \"345\",\n" +
+ " \"description\": \"Testing /json/docs srcField 2\",\n" +
+ "\n" +
+ " \"nested_data\" : {\n" +
+ " \"nested_inside\" : \"check check check 2\"\n" +
+ " }\n" +
+ "}";
+ JsonRecordReader streamer = JsonRecordReader.getInst("/", Arrays.asList("id:/id"));
+ RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json + json2));
+
+
+ streamer.streamRecords(parser, new JsonRecordReader.Handler() {
+ int count = 0;
+
+ @Override
+ public void handle(Map<String, Object> record, String path) {
+ count++;
+ String buf = parser.getBuf();
+ parser.resetBuf();
+
+ Map m = (Map) Utils.fromJSONString(buf);
+ if (count == 1) {
+ assertEquals(m.get("id"), "123");
+ assertEquals(m.get("description"), "Testing /json/docs srcField 1");
+ assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 1");
+ }
+ if (count++ == 2) {
+ assertEquals(m.get("id"), "345");
+ assertEquals(m.get("description"), "Testing /json/docs srcField 2");
+ assertEquals(((Map) m.get("nested_data")).get("nested_inside"), "check check check 2");
+ }
+ }
+ });
+
+ }
+
public void testRecursiveWildCard() throws IOException {
String json = "{\n" +
" \"a\":\"A\" ,\n" +
[2/2] lucene-solr:branch_6x: Merge remote-tracking branch
'origin/branch_6x' into branch_6x
Posted by no...@apache.org.
Merge remote-tracking branch 'origin/branch_6x' into branch_6x
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/99109075
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/99109075
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/99109075
Branch: refs/heads/branch_6x
Commit: 9910907551daaf32da00fbfa101fbdf31b49f282
Parents: 73d5f1c 4a71e03
Author: Noble Paul <no...@apache.org>
Authored: Tue Jun 21 18:52:15 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jun 21 18:52:15 2016 +0530
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
.../uk/UkrainianMorfologikAnalyzer.java | 154 +++
.../apache/lucene/analysis/uk/package-info.java | 21 +
.../org/apache/lucene/analysis/uk/stopwords.txt | 1269 ++++++++++++++++++
.../apache/lucene/analysis/uk/ukrainian.dict | Bin 0 -> 1707759 bytes
.../apache/lucene/analysis/uk/ukrainian.info | 10 +
.../analysis/uk/TestUkrainianAnalyzer.java | 65 +
.../lucene/store/MockDirectoryWrapper.java | 11 +
8 files changed, 1534 insertions(+)
----------------------------------------------------------------------