You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ge...@apache.org on 2019/10/08 10:32:46 UTC
[lucene-solr] 03/04: SOLR-13539: Fix MV removeregex atomic-updates
This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 5dc480f88f03e120c692e7695e4a80ef7e662113
Author: Jason Gerlowski <ge...@apache.org>
AuthorDate: Mon Oct 7 09:29:31 2019 -0400
SOLR-13539: Fix MV removeregex atomic-updates
Prior to this commit, the ByteArrayUtf8CharSequence issues had been
fixed on single value removeregex commands, but not if multiple regex's
were used.
This commit fixes our NamedList parsing for this additional case. It
also adds some tests for related atomic-update cases.
Co-Authored-By: Tim Owen
---
solr/CHANGES.txt | 3 +
.../processor/AtomicUpdateDocumentMerger.java | 6 +-
.../solr/update/processor/AtomicUpdatesTest.java | 108 ++++++++++++++++++++-
3 files changed, 113 insertions(+), 4 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8011512..5a01693 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -198,6 +198,9 @@ Bug Fixes
* SOLR-13790: LRUStatsCache size explosion and ineffective caching. (ab)
+* SOLR-13539: Fix for class-cast issues during atomic-update 'removeregex' operations. This also incorporated some
+ tests Tim wrote as a part of SOLR-9505. (Tim Owen via Jason Gerlowski)
+
Other Changes
----------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java b/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java
index ea42552..f0972db 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java
@@ -541,9 +541,9 @@ public class AtomicUpdateDocumentMerger {
private Collection<Pattern> preparePatterns(Object fieldVal) {
final Collection<Pattern> patterns = new LinkedHashSet<>(1);
if (fieldVal instanceof Collection) {
- Collection<String> patternVals = (Collection<String>) fieldVal;
- for (String patternVal : patternVals) {
- patterns.add(Pattern.compile(patternVal));
+ Collection<Object> patternVals = (Collection<Object>) fieldVal;
+ for (Object patternVal : patternVals) {
+ patterns.add(Pattern.compile(patternVal.toString()));
}
} else {
patterns.add(Pattern.compile(fieldVal.toString()));
diff --git a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
index 1240486..48c76b7 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java
@@ -75,6 +75,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "cat:ccc", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -88,6 +89,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "cat:ccc", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "21");
@@ -142,6 +144,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -155,6 +158,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "1021");
@@ -210,6 +214,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -223,6 +228,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "1021");
@@ -274,6 +280,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -287,6 +294,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "1021");
@@ -339,6 +347,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
doc.setField("id", "1001");
@@ -351,6 +360,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "1021");
@@ -423,6 +433,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -436,6 +447,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:222", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "intRemove:333", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "1021");
@@ -489,6 +501,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:22222222", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "intRemove:33333333", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -502,6 +515,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "intRemove:22222222", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "intRemove:33333333", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "1021");
@@ -559,6 +573,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "dateRemove:*", "indent", "true"), "//result[@numFound = '4']");
}
assertQ(req("q", "dateRemove:\"2014-09-02T12:00:00Z\"", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "dateRemove:\"2014-09-03T12:00:00Z\"", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
doc.setField("id", "10001");
@@ -672,6 +687,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "dateRemove:*", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "dateRemove:\"2014-09-02T12:00:00Z\"", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "dateRemove:\"2014-09-03T12:00:00Z\"", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "10021");
@@ -794,6 +810,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "floatRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "floatRemove:\"222.222\"", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "floatRemove:\"333.333\"", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -808,6 +825,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "floatRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "floatRemove:\"222.222\"", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "floatRemove:\"333.333\"", "indent", "true"), "//result[@numFound = '3']"); // remove only removed first occurrence
doc = new SolrInputDocument();
doc.setField("id", "10021");
@@ -832,7 +850,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "floatRemove:\"111.111\"", "indent", "true"), "//result[@numFound = '3']");
}
- @Test
+ @Test
public void testRemoveregex() throws Exception {
SolrInputDocument doc;
@@ -862,6 +880,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '3']");
+ assertQ(req("q", "cat:ccc", "indent", "true"), "//result[@numFound = '3']");
doc = new SolrInputDocument();
@@ -875,6 +894,7 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '4']");
assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '2']");
+ assertQ(req("q", "cat:ccc", "indent", "true"), "//result[@numFound = '2']"); // removeregex does remove all occurrences
doc = new SolrInputDocument();
doc.setField("id", "21");
@@ -900,6 +920,43 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
}
@Test
+ public void testRemoveregexMustMatchWholeValue() throws Exception {
+ SolrInputDocument doc;
+
+ doc = new SolrInputDocument();
+ doc.setField("id", "1");
+ doc.setField("cat", new String[]{"aaa", "bbb", "ccc", "ccc", "ddd"});
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '1']");
+
+
+ doc = new SolrInputDocument();
+ doc.setField("id", "1");
+ List<String> removeList = new ArrayList<>();
+ removeList.add("bb");
+ doc.setField("cat", ImmutableMap.of("removeregex", removeList)); //behavior when hitting Solr through ZK
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '1']"); // Was not removed - regex didn't match whole value
+
+ doc = new SolrInputDocument();
+ doc.setField("id", "1");
+ removeList = new ArrayList<>();
+ removeList.add("bbb");
+ doc.setField("cat", ImmutableMap.of("removeregex", removeList)); //behavior when hitting Solr through ZK
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '0']"); // Was removed now - regex matches
+ }
+
+ @Test
public void testAdd() throws Exception {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("id", "3");
@@ -976,6 +1033,55 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {
}
@Test
+ public void testAddMultiple() throws Exception {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", "3");
+ doc.setField("cat", new String[]{"aaa", "ccc"});
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '0']");
+
+
+ doc = new SolrInputDocument();
+ doc.setField("id", "3");
+ doc.setField("cat", ImmutableMap.of("add", "bbb"));
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '1']");
+
+ doc = new SolrInputDocument();
+ doc.setField("id", "3");
+ doc.setField("cat", ImmutableMap.of("add", "bbb"));
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '1']"); // Should now have 2 occurrences of bbb
+
+ doc = new SolrInputDocument();
+ doc.setField("id", "3");
+ doc.setField("cat", ImmutableMap.of("remove", "bbb"));
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '1']"); // remove only removed first occurrence
+
+ doc = new SolrInputDocument();
+ doc.setField("id", "3");
+ doc.setField("cat", ImmutableMap.of("remove", "bbb"));
+ assertU(adoc(doc));
+ assertU(commit());
+
+ assertQ(req("q", "cat:*", "indent", "true"), "//result[@numFound = '1']");
+ assertQ(req("q", "cat:bbb", "indent", "true"), "//result[@numFound = '0']"); // remove now removed last occurrence
+ }
+
+ @Test
public void testSet() throws Exception {
SolrInputDocument doc;