You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/02/12 18:26:34 UTC
[lucene-solr] 01/01: SOLR-14013: trying to port to SOlr 7.7
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr14013_solr7_7
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 3d17a424196f3bc35744d2fece3166b464131f03
Author: noble <no...@apache.org>
AuthorDate: Wed Feb 12 23:55:30 2020 +0530
SOLR-14013: trying to port to SOlr 7.7
---
.../LanguageIdentifierUpdateProcessor.java | 22 ++++-----
.../org/apache/solr/response/DocsStreamer.java | 7 +--
.../org/apache/solr/update/DocumentBuilder.java | 4 +-
.../java/org/apache/solr/common/SolrDocument.java | 11 ++---
.../solrj/embedded/SolrExampleJettyTest.java | 53 ++++++++++++++++------
5 files changed, 62 insertions(+), 35 deletions(-)
diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
index 3679905..93660dc 100644
--- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
+++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
@@ -16,6 +16,16 @@
*/
package org.apache.solr.update.processor;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.regex.Pattern;
+
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
@@ -29,16 +39,6 @@ import org.apache.solr.update.AddUpdateCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.regex.Pattern;
-
/**
* Identifies the language of a set of input fields.
@@ -230,7 +230,7 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro
}
} else {
// langField is set, we sanity check it against whitelist and fallback
- docLang = resolveLanguage((String) doc.getFieldValue(langField), fallbackLang);
+ docLang = resolveLanguage(doc.getFieldValue(langField).toString(), fallbackLang);
docLangs.add(docLang);
log.debug("Field "+langField+" already contained value "+docLang+", not overwriting.");
}
diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
index 3d1976e..c6010e1 100644
--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
@@ -49,8 +49,8 @@ import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
-import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrReturnFields;
/**
@@ -148,9 +148,10 @@ public class DocsStreamer implements Iterator<SolrDocument> {
// because that doesn't include extra fields needed by transformers
final Set<String> fieldNamesNeeded = fields.getLuceneFieldNames();
+ BinaryResponseWriter.MaskCharSeqSolrDocument masked = null;
final SolrDocument out = ResultContext.READASBYTES.get() == null ?
new SolrDocument() :
- new BinaryResponseWriter.MaskCharSeqSolrDocument();
+ (masked = new BinaryResponseWriter.MaskCharSeqSolrDocument());
// NOTE: it would be tempting to try and optimize this to loop over fieldNamesNeeded
// when it's smaller then the IndexableField[] in the Document -- but that's actually *less* effecient
@@ -160,7 +161,7 @@ public class DocsStreamer implements Iterator<SolrDocument> {
final String fname = f.name();
if (null == fieldNamesNeeded || fieldNamesNeeded.contains(fname) ) {
// Make sure multivalued fields are represented as lists
- Object existing = out.get(fname);
+ Object existing = masked == null ? out.get(fname) : masked.getRaw(fname);
if (existing == null) {
SchemaField sf = schema.getFieldOrNull(fname);
if (sf != null && sf.multiValued()) {
diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
index 90694ff..543a4cb 100644
--- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
+++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
@@ -196,8 +196,8 @@ public class DocumentBuilder {
// Perhaps trim the length of a copy field
Object val = v;
- if( val instanceof String && cf.getMaxChars() > 0 ) {
- val = cf.getLimitedValue((String)val);
+ if( val instanceof CharSequence && cf.getMaxChars() > 0 ) {
+ val = cf.getLimitedValue(val.toString());
}
addField(out, destinationField, val,
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
index 0563c9e..d7b268b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
@@ -28,8 +28,6 @@ import java.util.Set;
import org.apache.solr.common.util.NamedList;
-import static org.apache.solr.common.util.ByteArrayUtf8CharSequence.convertCharSeq;
-
/**
* A concrete representation of a document within a Solr index. Unlike a lucene
@@ -289,14 +287,14 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
/** Get the field Value */
@Override
public Object get(Object key) {
- return convertCharSeq(getFirstValue( (String)key));
+ return getFirstValue( (String)key);
}
// Easily Supported methods
@Override
public boolean containsKey(Object key) { return _fields.containsKey( key ); }
@Override
- public Set<String> keySet() { return (Set<String>) convertCharSeq(_fields.keySet()); }
+ public Set<String> keySet() { return _fields.keySet(); }
@Override
public int size() { return _fields.size(); }
@Override
@@ -368,7 +366,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
@Override
public Object remove(Object key) {
- return convertCharSeq(_fields.remove(key));
+ return _fields.remove(key);
}
@Override
@@ -378,7 +376,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
@Override
public Collection<Object> values() {
- return convertCharSeq(_fields.values());
+ return _fields.values();
}
@Override
@@ -409,6 +407,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
@Override
public int getChildDocumentCount() {
+ if (_childDocuments == null) return 0;
return _childDocuments.size();
}
}
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
index 5fe8c9b..b0aa04e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
@@ -19,9 +19,12 @@ package org.apache.solr.client.solrj.embedded;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
@@ -43,7 +46,8 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import org.noggit.ObjectBuilder;
+
+import static org.apache.solr.common.util.Utils.fromJSONString;
/**
* TODO? perhaps use:
@@ -85,7 +89,8 @@ public class SolrExampleJettyTest extends SolrExampleTests {
HttpClient httpClient = client.getHttpClient();
HttpPost post = new HttpPost(getUri(client));
post.setHeader("Content-Type", "application/json");
- post.setEntity(new InputStreamEntity(new ByteArrayInputStream(json.getBytes("UTF-8")), -1));
+ post.setEntity(new InputStreamEntity(
+ new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)), -1));
HttpResponse response = httpClient.execute(post, HttpClientUtil.createNewHttpClientRequestContext());
assertEquals(200, response.getStatusLine().getStatusCode());
client.commit();
@@ -94,13 +99,13 @@ public class SolrExampleJettyTest extends SolrExampleTests {
SolrDocument doc = rsp.getResults().get(0);
String src = (String) doc.getFieldValue("_src_");
- Map m = (Map) ObjectBuilder.fromJSON(src);
+ Map m = (Map) fromJSONString(src);
assertEquals("abc1",m.get("id"));
assertEquals("name1",m.get("name"));
doc = rsp.getResults().get(1);
src = (String) doc.getFieldValue("_src_");
- m = (Map) ObjectBuilder.fromJSON(src);
+ m = (Map) fromJSONString(src);
assertEquals("name2",m.get("name"));
}
@@ -112,6 +117,37 @@ public class SolrExampleJettyTest extends SolrExampleTests {
baseURL + "/update/json/docs";
}
+ private void runQueries(HttpSolrClient client, int count, boolean warmup) throws SolrServerException, IOException {
+ long start = System.nanoTime();
+ for (int i = 0; i < count; i++) {
+ client.query(new SolrQuery("*:*"));
+ }
+ if (warmup) return;
+ System.out.println("time taken : " + ((System.nanoTime() - start)) / (1000 * 1000));
+ }
+
+
+ @Test
+ public void testUtf8PerfDegradation() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ HttpSolrClient client = (HttpSolrClient) getSolrClient();
+
+ client.deleteByQuery("*:*");
+ client.commit();
+
+
+ doc.addField("id", "1");
+ doc.addField("b_is", IntStream.range(0, 30000).boxed().collect(Collectors.toList()));
+
+ client.add(doc);
+ client.commit();
+ long start = System.nanoTime();
+ QueryResponse rsp = client.query(new SolrQuery("*:*"));
+ System.out.println("time taken : " + ((System.nanoTime() - start)) / (1000 * 1000));
+ assertEquals(1, rsp.getResults().getNumFound());
+
+ }
+
@Ignore
public void testUtf8QueryPerf() throws Exception {
HttpSolrClient client = (HttpSolrClient) getSolrClient();
@@ -157,13 +193,4 @@ public class SolrExampleJettyTest extends SolrExampleTests {
}
- private void runQueries(HttpSolrClient client, int count, boolean warmup) throws SolrServerException, IOException {
- long start = System.nanoTime();
- for (int i = 0; i < count; i++) {
- client.query(new SolrQuery("*:*"));
- }
- if (warmup) return;
- System.out.println("time taken : " + ((System.nanoTime() - start)) / (1000 * 1000));
- }
-
}