You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/10/12 13:23:38 UTC
lucene-solr:branch_6x: LUCENE-7491: fix merge exception if the same
field has points in some segments but not in others
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x df4170629 -> 86b03358d
LUCENE-7491: fix merge exception if the same field has points in some segments but not in others
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/86b03358
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/86b03358
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/86b03358
Branch: refs/heads/branch_6x
Commit: 86b03358d59c584c89823e187b8806da48eb82af
Parents: df41706
Author: Mike McCandless <mi...@apache.org>
Authored: Wed Oct 12 09:23:25 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Wed Oct 12 09:23:25 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +++++
.../org/apache/lucene/codecs/PointsWriter.java | 7 +++++-
.../codecs/lucene60/Lucene60PointsWriter.java | 5 ++---
.../java/org/apache/lucene/index/FieldInfo.java | 4 ++++
.../lucene/index/BasePointsFormatTestCase.java | 23 ++++++++++++++++++++
5 files changed, 40 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86b03358/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9a29b7b..1e17025 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -35,6 +35,11 @@ Bug Fixes
* LUCENE-7486: DisjunctionMaxQuery does not work correctly with queries that
return negative scores. (Ivan Provalov, Uwe Schindler, Adrien Grand)
+* LUCENE-7491: Suddenly turning on dimensional points for some fields
+ that already exist in an index but didn't previously index
+ dimensional points could cause unexpected merge exceptions (Hans
+ Lund, Mike McCandless)
+
Improvements
* LUCENE-7439: FuzzyQuery now matches all terms within the specified
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86b03358/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
index 05084db..eac1716 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
@@ -47,7 +47,7 @@ public abstract class PointsWriter implements Closeable {
PointsReader pointsReader = mergeState.pointsReaders[i];
if (pointsReader != null) {
FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null) {
+ if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
maxPointCount += pointsReader.size(fieldInfo.name);
docCount += pointsReader.getDocCount(fieldInfo.name);
}
@@ -75,6 +75,11 @@ public abstract class PointsWriter implements Closeable {
continue;
}
+ if (readerFieldInfo.getPointDimensionCount() == 0) {
+ // This segment saw this field, but the field did not index points in it:
+ continue;
+ }
+
MergeState.DocMap docMap = mergeState.docMaps[i];
pointsReader.intersect(fieldInfo.name,
new IntersectVisitor() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86b03358/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
index ff9de58..05a1bf9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java
@@ -165,7 +165,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
if (reader != null) {
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null) {
+ if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
totMaxSize += reader.size(fieldInfo.name);
singleValuePerDoc &= reader.size(fieldInfo.name) == reader.getDocCount(fieldInfo.name);
}
@@ -200,10 +200,9 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable {
// reader's FieldInfo as we do below) because field numbers can easily be different
// when addIndexes(Directory...) copies over segments from another index:
-
FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null) {
+ if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
BKDReader bkdReader = reader60.readers.get(readerFieldInfo.number);
if (bkdReader != null) {
bkdReaders.add(bkdReader);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86b03358/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
index 57cab47..1118f27 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
@@ -145,6 +145,8 @@ public final class FieldInfo {
if (this.pointDimensionCount == 0 && dimensionCount != 0) {
this.pointDimensionCount = dimensionCount;
this.pointNumBytes = dimensionNumBytes;
+ } else if (this.pointDimensionCount != dimensionCount || this.pointNumBytes != dimensionNumBytes) {
+ throw new IllegalArgumentException("cannot change field \"" + name + "\" from points dimensionCount=" + this.pointDimensionCount + ", numBytes=" + this.pointNumBytes + " to inconsistent dimensionCount=" + dimensionCount + ", numBytes=" + dimensionNumBytes);
}
if (this.indexOptions != IndexOptions.NONE) { // if updated field data is not for indexing, leave the updates out
@@ -187,6 +189,8 @@ public final class FieldInfo {
pointDimensionCount = count;
pointNumBytes = numBytes;
+
+ assert checkConsistency();
}
/** Return point dimension count */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86b03358/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
index 93dd55d..ac0106f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointsFormatTestCase.java
@@ -981,4 +981,27 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa
// structure than the tree created by adding points separately
return false;
}
+
+ // LUCENE-7491
+ public void testMixedSchema() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ iwc.setMaxBufferedDocs(2);
+ for(int i=0;i<2;i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));
+ doc.add(new IntPoint("int", i));
+ w.addDocument(doc);
+ }
+ // index has 1 segment now (with 2 docs) and that segment does have points, but the "id" field in particular does NOT
+
+ Document doc = new Document();
+ doc.add(new IntPoint("id", 0));
+ w.addDocument(doc);
+ // now we write another segment where the id field does have points:
+
+ w.forceMerge(1);
+ IOUtils.close(w, dir);
+ }
}