You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/02/12 18:26:33 UTC
[lucene-solr] branch jira/solr14013_solr7_7 created (now 3d17a42)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a change to branch jira/solr14013_solr7_7
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
at 3d17a42 SOLR-14013: trying to port to SOlr 7.7
This branch includes the following new commits:
new 3d17a42 SOLR-14013: trying to port to SOlr 7.7
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[lucene-solr] 01/01: SOLR-14013: trying to port to SOlr 7.7
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr14013_solr7_7
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 3d17a424196f3bc35744d2fece3166b464131f03
Author: noble <no...@apache.org>
AuthorDate: Wed Feb 12 23:55:30 2020 +0530
SOLR-14013: trying to port to SOlr 7.7
---
.../LanguageIdentifierUpdateProcessor.java | 22 ++++-----
.../org/apache/solr/response/DocsStreamer.java | 7 +--
.../org/apache/solr/update/DocumentBuilder.java | 4 +-
.../java/org/apache/solr/common/SolrDocument.java | 11 ++---
.../solrj/embedded/SolrExampleJettyTest.java | 53 ++++++++++++++++------
5 files changed, 62 insertions(+), 35 deletions(-)
diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
index 3679905..93660dc 100644
--- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
+++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
@@ -16,6 +16,16 @@
*/
package org.apache.solr.update.processor;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.regex.Pattern;
+
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
@@ -29,16 +39,6 @@ import org.apache.solr.update.AddUpdateCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.regex.Pattern;
-
/**
* Identifies the language of a set of input fields.
@@ -230,7 +230,7 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro
}
} else {
// langField is set, we sanity check it against whitelist and fallback
- docLang = resolveLanguage((String) doc.getFieldValue(langField), fallbackLang);
+ docLang = resolveLanguage(doc.getFieldValue(langField).toString(), fallbackLang);
docLangs.add(docLang);
log.debug("Field "+langField+" already contained value "+docLang+", not overwriting.");
}
diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
index 3d1976e..c6010e1 100644
--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
@@ -49,8 +49,8 @@ import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
-import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrReturnFields;
/**
@@ -148,9 +148,10 @@ public class DocsStreamer implements Iterator<SolrDocument> {
// because that doesn't include extra fields needed by transformers
final Set<String> fieldNamesNeeded = fields.getLuceneFieldNames();
+ BinaryResponseWriter.MaskCharSeqSolrDocument masked = null;
final SolrDocument out = ResultContext.READASBYTES.get() == null ?
new SolrDocument() :
- new BinaryResponseWriter.MaskCharSeqSolrDocument();
+ (masked = new BinaryResponseWriter.MaskCharSeqSolrDocument());
// NOTE: it would be tempting to try and optimize this to loop over fieldNamesNeeded
// when it's smaller then the IndexableField[] in the Document -- but that's actually *less* effecient
@@ -160,7 +161,7 @@ public class DocsStreamer implements Iterator<SolrDocument> {
final String fname = f.name();
if (null == fieldNamesNeeded || fieldNamesNeeded.contains(fname) ) {
// Make sure multivalued fields are represented as lists
- Object existing = out.get(fname);
+ Object existing = masked == null ? out.get(fname) : masked.getRaw(fname);
if (existing == null) {
SchemaField sf = schema.getFieldOrNull(fname);
if (sf != null && sf.multiValued()) {
diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
index 90694ff..543a4cb 100644
--- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
+++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
@@ -196,8 +196,8 @@ public class DocumentBuilder {
// Perhaps trim the length of a copy field
Object val = v;
- if( val instanceof String && cf.getMaxChars() > 0 ) {
- val = cf.getLimitedValue((String)val);
+ if( val instanceof CharSequence && cf.getMaxChars() > 0 ) {
+ val = cf.getLimitedValue(val.toString());
}
addField(out, destinationField, val,
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
index 0563c9e..d7b268b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
@@ -28,8 +28,6 @@ import java.util.Set;
import org.apache.solr.common.util.NamedList;
-import static org.apache.solr.common.util.ByteArrayUtf8CharSequence.convertCharSeq;
-
/**
* A concrete representation of a document within a Solr index. Unlike a lucene
@@ -289,14 +287,14 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
/** Get the field Value */
@Override
public Object get(Object key) {
- return convertCharSeq(getFirstValue( (String)key));
+ return getFirstValue( (String)key);
}
// Easily Supported methods
@Override
public boolean containsKey(Object key) { return _fields.containsKey( key ); }
@Override
- public Set<String> keySet() { return (Set<String>) convertCharSeq(_fields.keySet()); }
+ public Set<String> keySet() { return _fields.keySet(); }
@Override
public int size() { return _fields.size(); }
@Override
@@ -368,7 +366,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
@Override
public Object remove(Object key) {
- return convertCharSeq(_fields.remove(key));
+ return _fields.remove(key);
}
@Override
@@ -378,7 +376,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
@Override
public Collection<Object> values() {
- return convertCharSeq(_fields.values());
+ return _fields.values();
}
@Override
@@ -409,6 +407,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
@Override
public int getChildDocumentCount() {
+ if (_childDocuments == null) return 0;
return _childDocuments.size();
}
}
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
index 5fe8c9b..b0aa04e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
@@ -19,9 +19,12 @@ package org.apache.solr.client.solrj.embedded;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
@@ -43,7 +46,8 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import org.noggit.ObjectBuilder;
+
+import static org.apache.solr.common.util.Utils.fromJSONString;
/**
* TODO? perhaps use:
@@ -85,7 +89,8 @@ public class SolrExampleJettyTest extends SolrExampleTests {
HttpClient httpClient = client.getHttpClient();
HttpPost post = new HttpPost(getUri(client));
post.setHeader("Content-Type", "application/json");
- post.setEntity(new InputStreamEntity(new ByteArrayInputStream(json.getBytes("UTF-8")), -1));
+ post.setEntity(new InputStreamEntity(
+ new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)), -1));
HttpResponse response = httpClient.execute(post, HttpClientUtil.createNewHttpClientRequestContext());
assertEquals(200, response.getStatusLine().getStatusCode());
client.commit();
@@ -94,13 +99,13 @@ public class SolrExampleJettyTest extends SolrExampleTests {
SolrDocument doc = rsp.getResults().get(0);
String src = (String) doc.getFieldValue("_src_");
- Map m = (Map) ObjectBuilder.fromJSON(src);
+ Map m = (Map) fromJSONString(src);
assertEquals("abc1",m.get("id"));
assertEquals("name1",m.get("name"));
doc = rsp.getResults().get(1);
src = (String) doc.getFieldValue("_src_");
- m = (Map) ObjectBuilder.fromJSON(src);
+ m = (Map) fromJSONString(src);
assertEquals("name2",m.get("name"));
}
@@ -112,6 +117,37 @@ public class SolrExampleJettyTest extends SolrExampleTests {
baseURL + "/update/json/docs";
}
+ private void runQueries(HttpSolrClient client, int count, boolean warmup) throws SolrServerException, IOException {
+ long start = System.nanoTime();
+ for (int i = 0; i < count; i++) {
+ client.query(new SolrQuery("*:*"));
+ }
+ if (warmup) return;
+ System.out.println("time taken : " + ((System.nanoTime() - start)) / (1000 * 1000));
+ }
+
+
+ @Test
+ public void testUtf8PerfDegradation() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ HttpSolrClient client = (HttpSolrClient) getSolrClient();
+
+ client.deleteByQuery("*:*");
+ client.commit();
+
+
+ doc.addField("id", "1");
+ doc.addField("b_is", IntStream.range(0, 30000).boxed().collect(Collectors.toList()));
+
+ client.add(doc);
+ client.commit();
+ long start = System.nanoTime();
+ QueryResponse rsp = client.query(new SolrQuery("*:*"));
+ System.out.println("time taken : " + ((System.nanoTime() - start)) / (1000 * 1000));
+ assertEquals(1, rsp.getResults().getNumFound());
+
+ }
+
@Ignore
public void testUtf8QueryPerf() throws Exception {
HttpSolrClient client = (HttpSolrClient) getSolrClient();
@@ -157,13 +193,4 @@ public class SolrExampleJettyTest extends SolrExampleTests {
}
- private void runQueries(HttpSolrClient client, int count, boolean warmup) throws SolrServerException, IOException {
- long start = System.nanoTime();
- for (int i = 0; i < count; i++) {
- client.query(new SolrQuery("*:*"));
- }
- if (warmup) return;
- System.out.println("time taken : " + ((System.nanoTime() - start)) / (1000 * 1000));
- }
-
}