You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/11/17 08:33:35 UTC
[14/40] lucene-solr:jira/solr-11285-sim: SOLR-11231: Guard against
unset fields when performing language detection. This closes #232
SOLR-11231: Guard against unset fields when performing language detection. This closes #232
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c3513e92
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c3513e92
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c3513e92
Branch: refs/heads/jira/solr-11285-sim
Commit: c3513e928141ad9ef0aa4e4fef641c70d0e4b4be
Parents: 3045349
Author: Steve Rowe <sa...@apache.org>
Authored: Mon Nov 13 17:57:24 2017 -0500
Committer: Steve Rowe <sa...@apache.org>
Committed: Mon Nov 13 17:57:24 2017 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../TikaLanguageIdentifierUpdateProcessor.java | 18 +++++++++++++-----
...eIdentifierUpdateProcessorFactoryTestCase.java | 13 +++++++++++++
3 files changed, 29 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3513e92/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d151353..f872b96 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -105,6 +105,9 @@ Bug Fixes
* SOLR-11619: V2 requests that needed to be forwarded to other nodes would get an NPE. (David Smiley)
+* SOLR-11231: Guard against unset fields when performing language detection.
+ (Chris Beer via Steve Rowe)
+
Optimizations
----------------------
* SOLR-11285: Refactor autoscaling framework to avoid direct references to Zookeeper and Solr
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3513e92/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java
index 836a3bf..df0e5f7 100644
--- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java
+++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/TikaLanguageIdentifierUpdateProcessor.java
@@ -112,13 +112,21 @@ public class TikaLanguageIdentifierUpdateProcessor extends LanguageIdentifierUpd
private int getExpectedSize(SolrInputDocument doc, String[] fields) {
int docSize = 0;
for (String field : fields) {
- Collection<Object> contents = doc.getFieldValues(field);
- for (Object content : contents) {
- if (content instanceof String) {
- docSize += Math.min(((String) content).length(), maxFieldValueChars);
+ if (doc.containsKey(field)) {
+ Collection<Object> contents = doc.getFieldValues(field);
+ if (contents != null) {
+ for (Object content : contents) {
+ if (content instanceof String) {
+ docSize += Math.min(((String) content).length(), maxFieldValueChars);
+ }
+ }
+
+ if (docSize > maxTotalChars) {
+ docSize = maxTotalChars;
+ break;
+ }
}
}
- docSize = Math.min(docSize, maxTotalChars);
}
return docSize;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3513e92/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
----------------------------------------------------------------------
diff --git a/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java b/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
index c7381a9..b90f54a 100644
--- a/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
+++ b/solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
@@ -212,6 +212,19 @@ public abstract class LanguageIdentifierUpdateProcessorFactoryTestCase extends S
}
@Test
+ public void testMissingFieldEmptyString() throws Exception {
+ SolrInputDocument doc;
+ ModifiableSolrParams parameters = new ModifiableSolrParams();
+ parameters.add("langid.fl", "no_such_field");
+ parameters.add("langid.langField", "language");
+ parameters.add("langid.enforceSchema", "false");
+ liProcessor = createLangIdProcessor(parameters);
+
+ doc = new SolrInputDocument();
+ assertEquals("", liProcessor.process(doc).getFieldValue("language"));
+ }
+
+ @Test
public void testFallback() throws Exception {
SolrInputDocument doc;
ModifiableSolrParams parameters = new ModifiableSolrParams();