svn commit: r1423711 - in /lucene/dev/branches/lucene_solr_3_6/lucene: CHANGES.txt core/src/java/org/apache/lucene/util/PagedBytes.java core/src/test/org/apache/lucene/util/Test2BPagedBytes.java

[mi...@apache.org]

Author: mikemccand
Date: Tue Dec 18 23:13:45 2012
New Revision: 1423711

URL: http://svn.apache.org/viewvc?rev=1423711&view=rev
Log:
LUCENE-4635: fix overflow bug causing AIOOBE when index has many unique terms

Added:
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java   (with props)
Modified:
    lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt?rev=1423711&r1=1423710&r2=1423711&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt Tue Dec 18 23:13:45 2012
@@ -37,6 +37,10 @@ Bug Fixes:
 * LUCENE-4411: when sampling is enabled for a FacetRequest, its depth
   parameter is reset to the default (1), even if set otherwise.
   (Gilad Barkai via Shai Erera)
+
+* LUCENE-4635: Fixed ArrayIndexOutOfBoundsException when in-memory
+  terms index requires more than 2.1 GB RAM (indices with billions of
+  terms).  (Tom Burton-West via Mike McCandless)
   
 Documentation
 

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java?rev=1423711&r1=1423710&r2=1423711&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java Tue Dec 18 23:13:45 2012
@@ -411,7 +411,7 @@ public final class PagedBytes {
 
     /** Returns the current byte position. */
     public long getPosition() {
-      return currentBlockIndex * blockSize + currentBlockUpto;
+      return ((long) currentBlockIndex * blockSize) + currentBlockUpto;
     }
   
     /** Seek to a position previously obtained from
@@ -516,7 +516,7 @@ public final class PagedBytes {
       if (currentBlock == null) {
         return 0;
       } else {
-        return blocks.size() * blockSize + upto;
+        return ((long) blocks.size() * blockSize) + upto;
       }
     }
   }

Added: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java?rev=1423711&view=auto
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java (added)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java Tue Dec 18 23:13:45 2012
@@ -0,0 +1,68 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Arrays;
+import java.util.Random;
+
+import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
+import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput;
+import org.junit.Ignore;
+
+@Ignore("You must increase heap to > 2 G to run this")
+public class Test2BPagedBytes extends LuceneTestCase {
+
+  public void test() throws Exception {
+    PagedBytes pb = new PagedBytes(15);
+    PagedBytesDataOutput dataOutput = pb.getDataOutput();
+    long netBytes = 0;
+    long seed = random.nextLong();
+    long lastFP = 0;
+    Random r2 = new Random(seed);
+    while(netBytes < 1.1*Integer.MAX_VALUE) {
+      int numBytes = _TestUtil.nextInt(r2, 1, 100000);
+      byte[] bytes = new byte[numBytes];
+      r2.nextBytes(bytes);
+      dataOutput.writeBytes(bytes, bytes.length);
+      long fp = dataOutput.getPosition();
+      assert fp == lastFP + numBytes;
+      lastFP = fp;
+      netBytes += numBytes;
+    }
+    pb.freeze(true);
+
+    PagedBytesDataInput dataInput = pb.getDataInput();
+    lastFP = 0;
+    r2 = new Random(seed);
+    netBytes = 0;
+    while(netBytes < 1.1*Integer.MAX_VALUE) {
+      int numBytes = _TestUtil.nextInt(r2, 1, 100000);
+      byte[] bytes = new byte[numBytes];
+      r2.nextBytes(bytes);
+
+      byte[] bytesIn = new byte[numBytes];
+      dataInput.readBytes(bytesIn, 0, numBytes);
+      assertTrue(Arrays.equals(bytes, bytesIn));
+
+      long fp = dataInput.getPosition();
+      assert fp == lastFP + numBytes;
+      lastFP = fp;
+      netBytes += numBytes;
+    }
+  }
+}