svn commit: r1438195 - in /lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene: codecs/lucene40/Lucene40DocValuesReader.java index/CheckIndex.java

[rm...@apache.org]

Author: rmuir
Date: Thu Jan 24 21:07:36 2013
New Revision: 1438195

URL: http://svn.apache.org/viewvc?rev=1438195&view=rev
Log:
LUCENE-4717: detect and mask bug in lucene4.0/4.1 indexes

Modified:
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
    lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java?rev=1438195&r1=1438194&r2=1438195&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java Thu Jan 24 21:07:36 2013
@@ -499,7 +499,7 @@ class Lucene40DocValuesReader extends Do
     data.readBytes(bytes, 0, bytes.length);
     final PackedInts.Reader reader = PackedInts.getReader(index);
     
-    return new SortedDocValues() {
+    return correctBuggyOrds(new SortedDocValues() {
       @Override
       public int getOrd(int docID) {
         return (int) reader.get(docID);
@@ -516,7 +516,7 @@ class Lucene40DocValuesReader extends Do
       public int getValueCount() {
         return valueCount;
       }
-    };
+    });
   }
   
   private SortedDocValues loadBytesVarSorted(FieldInfo field, IndexInput data, IndexInput index) throws IOException {
@@ -537,7 +537,7 @@ class Lucene40DocValuesReader extends Do
     
     final int valueCount = addressReader.size() - 1;
     
-    return new SortedDocValues() {
+    return correctBuggyOrds(new SortedDocValues() {
       @Override
       public int getOrd(int docID) {
         return (int)ordsReader.get(docID);
@@ -556,6 +556,34 @@ class Lucene40DocValuesReader extends Do
       public int getValueCount() {
         return valueCount;
       }
+    });
+  }
+  
+  // detects and corrects LUCENE-4717 in old indexes
+  private SortedDocValues correctBuggyOrds(final SortedDocValues in) {
+    final int maxDoc = state.segmentInfo.getDocCount();
+    for (int i = 0; i < maxDoc; i++) {
+      if (in.getOrd(i) == 0) {
+        return in; // ok
+      }
+    }
+    
+    // we had ord holes, return an ord-shifting-impl that corrects the bug
+    return new SortedDocValues() {
+      @Override
+      public int getOrd(int docID) {
+        return in.getOrd(docID) - 1;
+      }
+
+      @Override
+      public void lookupOrd(int ord, BytesRef result) {
+        in.lookupOrd(ord+1, result);
+      }
+
+      @Override
+      public int getValueCount() {
+        return in.getValueCount() - 1;
+      }
     };
   }
   

Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1438195&r1=1438194&r2=1438195&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Thu Jan 24 21:07:36 2013
@@ -1339,8 +1339,7 @@ public class CheckIndex {
       throw new RuntimeException("dv for field: " + fieldName + " reports wrong maxOrd=" + maxOrd + " but this is not the case: " + maxOrd2);
     }
     if (seenOrds.cardinality() != dv.getValueCount()) {
-      // nocommit: looks like maybe a bug in 4.0 indexes? dig on this (and think if we can correct it somehow...)
-      // throw new RuntimeException("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.getValueCount() + " but only used: " + seenOrds.cardinality());
+      throw new RuntimeException("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.getValueCount() + " but only used: " + seenOrds.cardinality());
     }
     BytesRef lastValue = null;
     BytesRef scratch = new BytesRef();