You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/05/22 19:58:50 UTC
[13/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8316: Allow DV
updates for not existing fields
LUCENE-8316: Allow DV updates for not existing fields
Today we prevent DV updates for non-existing fields except
of the soft deletes case. Yet, this can cause inconsitent field numbers
etc. since we don't go through the global field number map etc. This
change removes the limitation of updating DVs in docs even if the field
doesn't exists. This also has the benefit that the error messages if
the field type doesn't match is consistent with what DWPT throws.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0c362892
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0c362892
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0c362892
Branch: refs/heads/jira/solr-11779
Commit: 0c3628920afdc27bbaf1c057bf6519319ea78e51
Parents: 35a815b
Author: Simon Willnauer <si...@apache.org>
Authored: Wed May 16 13:20:22 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Wed May 16 19:59:20 2018 +0200
----------------------------------------------------------------------
.../org/apache/lucene/index/FieldInfos.java | 2 +-
.../org/apache/lucene/index/IndexWriter.java | 12 ++---
.../lucene/index/TestMixedDocValuesUpdates.java | 52 +++++++++++++++++++-
3 files changed, 57 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c362892/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index 78e8e14..4b472a5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -346,7 +346,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
synchronized Set<String> getFieldNames() {
- return Collections.unmodifiableSet(new HashSet<String>(nameToNumber.keySet()));
+ return Collections.unmodifiableSet(new HashSet<>(nameToNumber.keySet()));
}
synchronized void clear() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c362892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 27d6459..af2fd47 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1775,8 +1775,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
if (dvType == DocValuesType.NONE) {
throw new IllegalArgumentException("can only update NUMERIC or BINARY fields! field=" + f.name());
}
- if (!globalFieldNumberMap.contains(f.name(), dvType) && f.name().equals(config.softDeletesField) == false) {
- throw new IllegalArgumentException("can only update existing docvalues fields! field=" + f.name() + ", type=" + dvType);
+ if (globalFieldNumberMap.contains(f.name(), dvType) == false) {
+ // if this field doesn't exists we try to add it. if it exists and the DV type doesn't match we
+ // get a consistent error message as if you try to do that during an indexing operation.
+ globalFieldNumberMap.addOrGet(f.name(), -1, IndexOptions.NONE, dvType, 0, 0);
+ assert globalFieldNumberMap.contains(f.name(), dvType);
}
if (config.getIndexSortFields().contains(f.name())) {
throw new IllegalArgumentException("cannot update docvalues field involved in the index sort, field=" + f.name() + ", sort=" + config.getIndexSort());
@@ -1808,11 +1811,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
}
// for test purpose
- final synchronized Collection<String> getIndexFileNames() throws IOException {
- return segmentInfos.files(true);
- }
-
- // for test purpose
final synchronized int maxDoc(int i) {
if (i >= 0 && i < segmentInfos.size()) {
return segmentInfos.info(i).info.maxDoc();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c362892/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
index 4315af2..e3f34c0 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
@@ -639,6 +639,56 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
}
IOUtils.close(writer, dir);
}
-
+
+ public void testUpdateNotExistingFieldDV() throws IOException {
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "1", Store.YES));
+ doc.add(new NumericDocValuesField("test", 1));
+ writer.addDocument(doc);
+ if (random().nextBoolean()) {
+ writer.commit();
+ }
+ writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("not_existing", 1));
+
+ Document doc1 = new Document();
+ doc1.add(new StringField("id", "2", Store.YES));
+ doc1.add(new BinaryDocValuesField("not_existing", new BytesRef()));
+ IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+ writer.addDocument(doc1)
+ );
+ assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
+
+ iae = expectThrows(IllegalArgumentException.class, () ->
+ writer.updateDocValues(new Term("id", "1"), new BinaryDocValuesField("not_existing", new BytesRef()))
+ );
+ assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
+ }
+ }
+
+ public void testUpdateFieldWithNoPreviousDocValues() throws IOException {
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "1", Store.YES));
+ writer.addDocument(doc);
+ if (random().nextBoolean()) {
+ try (DirectoryReader reader = writer.getReader()) {
+ NumericDocValues id = reader.leaves().get(0).reader().getNumericDocValues("id");
+ assertNull(id);
+ }
+ } else if (random().nextBoolean()) {
+ writer.commit();
+ }
+ writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("id", 1));
+ try (DirectoryReader reader = writer.getReader()) {
+ NumericDocValues id = reader.leaves().get(0).reader().getNumericDocValues("id");
+ assertNotNull(id);
+ assertTrue(id.advanceExact(0));
+ assertEquals(1, id.longValue());
+ }
+ }
+ }
}