svn commit: r1687792 - in /lucene/dev/branches/branch_5x: ./ lucene/ lucene/CHANGES.txt lucene/core/ lucene/core/src/java/org/apache/lucene/index/FieldInfos.java

[mi...@apache.org]

Author: mikemccand
Date: Fri Jun 26 16:01:17 2015
New Revision: 1687792

URL: http://svn.apache.org/r1687792
Log:
LUCENE-6325: use array for number -> FieldInfo lookup, except in very sparse cases

Modified:
    lucene/dev/branches/branch_5x/   (props changed)
    lucene/dev/branches/branch_5x/lucene/   (props changed)
    lucene/dev/branches/branch_5x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_5x/lucene/core/   (props changed)
    lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java

Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1687792&r1=1687791&r2=1687792&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Fri Jun 26 16:01:17 2015
@@ -189,6 +189,10 @@ Optimizations
   with a filter of "baz" will internally leapfrog foo,bar,baz as one
   conjunction. (Ryan Ernst, Robert Muir, Adrien Grande)
 
+* LUCENE-6325: Reduce RAM usage of FieldInfos, and speed up lookup by
+  number, by using an array instead of TreeMap except in very sparse
+  cases (Robert Muir, Mike McCandless)
+
 Build
 
 * LUCENE-6518: Don't report false thread leaks from IBM J9

Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1687792&r1=1687791&r2=1687792&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Fri Jun 26 16:01:17 2015
@@ -25,6 +25,8 @@ import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import org.apache.lucene.util.ArrayUtil;
+
 /** 
  * Collection of {@link FieldInfo}s (accessible by number or by name).
  *  @lucene.experimental
@@ -38,7 +40,10 @@ public class FieldInfos implements Itera
   private final boolean hasNorms;
   private final boolean hasDocValues;
   
-  private final SortedMap<Integer,FieldInfo> byNumber = new TreeMap<>();
+  // used only by fieldInfo(int)
+  private final FieldInfo[] byNumberTable; // contiguous
+  private final SortedMap<Integer,FieldInfo> byNumberMap; // sparse
+  
   private final HashMap<String,FieldInfo> byName = new HashMap<>();
   private final Collection<FieldInfo> values; // for an unmodifiable iterator
   
@@ -54,6 +59,7 @@ public class FieldInfos implements Itera
     boolean hasNorms = false;
     boolean hasDocValues = false;
     
+    TreeMap<Integer, FieldInfo> byNumber = new TreeMap<>();
     for (FieldInfo info : infos) {
       if (info.number < 0) {
         throw new IllegalArgumentException("illegal field number: " + info.number + " for field " + info.name);
@@ -84,6 +90,22 @@ public class FieldInfos implements Itera
     this.hasNorms = hasNorms;
     this.hasDocValues = hasDocValues;
     this.values = Collections.unmodifiableCollection(byNumber.values());
+    Integer max = byNumber.isEmpty() ? null : Collections.max(byNumber.keySet());
+    
+    // Only usee TreeMap in the very sparse case (< 1/16th of the numbers are used),
+    // because TreeMap uses ~ 64 (32 bit JVM) or 120 (64 bit JVM w/o compressed oops)
+    // overall bytes per entry, but array uses 4 (32 bit JMV) or 8
+    // (64 bit JVM w/o compressed oops):
+    if (max != null && max < ArrayUtil.MAX_ARRAY_LENGTH && max < 16L*byNumber.size()) {
+      byNumberMap = null;
+      byNumberTable = new FieldInfo[max+1];
+      for (Map.Entry<Integer,FieldInfo> entry : byNumber.entrySet()) {
+        byNumberTable[entry.getKey()] = entry.getValue();
+      }
+    } else {
+      byNumberMap = byNumber;
+      byNumberTable = null;
+    }
   }
   
   /** Returns true if any fields have freqs */
@@ -123,8 +145,7 @@ public class FieldInfos implements Itera
   
   /** Returns the number of fields */
   public int size() {
-    assert byNumber.size() == byName.size();
-    return byNumber.size();
+    return byName.size();
   }
   
   /**
@@ -157,7 +178,14 @@ public class FieldInfos implements Itera
     if (fieldNumber < 0) {
       throw new IllegalArgumentException("Illegal field number: " + fieldNumber);
     }
-    return byNumber.get(fieldNumber);
+    if (byNumberTable != null) {
+      if (fieldNumber >= byNumberTable.length) {
+        return null;
+      }
+      return byNumberTable[fieldNumber];
+    } else {
+      return byNumberMap.get(fieldNumber);
+    }
   }
   
   static final class FieldNumbers {