svn commit: r1652622 - in /lucene/dev/branches/branch_5x: ./ solr/ solr/CHANGES.txt solr/contrib/ solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java

[no...@apache.org]

Author: noble
Date: Sat Jan 17 15:08:31 2015
New Revision: 1652622

URL: http://svn.apache.org/r1652622
Log:
SOLR-5147 addressing the OOM failures

Modified:
    lucene/dev/branches/branch_5x/   (props changed)
    lucene/dev/branches/branch_5x/solr/   (props changed)
    lucene/dev/branches/branch_5x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_5x/solr/contrib/   (props changed)
    lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java

Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1652622&r1=1652621&r2=1652622&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Sat Jan 17 15:08:31 2015
@@ -265,7 +265,8 @@ New Features
 
 * SOLR-6937: In schemaless mode ,replace spaces and special characters with underscore (Noble Paul)
 
-* SOLR-5147: Support child documents in DIH (Shawn Heisey, Thomas Champagne, Mikhail Khludnev via Noble Paul
+* SOLR-5147: Support child documents in DIH
+  (Vadim Kirilchuk, Shawn Heisey, Thomas Champagne, Mikhail Khludnev via Noble Paul)
   
 
 Bug Fixes

Modified: lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java?rev=1652622&r1=1652621&r2=1652622&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestHierarchicalDocBuilder.java Sat Jan 17 15:08:31 2015
@@ -56,14 +56,38 @@ public class TestHierarchicalDocBuilder
   private int id = 0; //unique id
   private SolrQueryRequest req;
   
+  /**
+   * Holds the data related to randomly created index.
+   * It is used for making assertions.
+   */
   private static class ContextHolder {
+    /** Overall documents number **/
     int counter = 0;
+    
+    /**
+     * Each Hierarchy object represents nested documents with a parent at the root of hierarchy
+     */
     List<Hierarchy> hierarchies = new ArrayList<Hierarchy>();
   }
   
+  /**
+   * Represents a hierarchical document structure
+   */
   private static class Hierarchy {
+    
+    /**
+     * Type of element, i.e. parent, child, grandchild, etc..
+     */
     String elementType;
+    
+    /**
+     * Fields of a current element
+     */
     Map<String, Object> elementData = new HashMap<String,Object>();
+    
+    /**
+     * Nested elements/documents hierarchies. 
+     */
     List<Hierarchy> elements = new ArrayList<Hierarchy>();
   }
   
@@ -167,8 +191,11 @@ public class TestHierarchicalDocBuilder
   public void testRandomDepthHierarchy() throws Exception {
     final String parentType = "parent";
     
-    int parentsNum = atLeast(5);
-    int depth = atLeast(3);
+    // Be aware that hierarchies grows exponentially, thus 
+    // numbers bigger than 6 may lead to significant memory usage
+    // and cause OOME
+    int parentsNum = 2 + random().nextInt(3);
+    int depth = 2 + random().nextInt(3);
     
     ContextHolder holder = new ContextHolder();
     
@@ -250,11 +277,59 @@ public class TestHierarchicalDocBuilder
     return ids;
   }
   
-  /** Internally configures MockDataSource **/
+  /**
+   * Creates randomized configuration of a specified depth. Simple configuration example:
+   * 
+   * <pre>
+   * 
+   * &lt;dataConfig>
+   *   <dataSource type="MockDataSource" />
+   *   &lt;document>
+   *     &lt;entity name="parent" query="SELECT * FROM parent">
+   *       &lt;field column="id" />
+   *       &lt;field column="desc" />
+   *       &lt;field column="type_s" />
+   *       &lt;entity child="true" name="parentChild0" query="select * from parentChild0 where parentChild0_parent_id='${parent.id}'">
+   *         &lt;field column="id" />
+   *         &lt;field column="desc" />
+   *         &lt;field column="type_s" />
+   *         &lt;entity child="true" name="parentChild0Child0" query="select * from parentChild0Child0 where parentChild0Child0_parent_id='${parentChild0.id}'">
+   *           &lt;field column="id" />
+   *           &lt;field column="desc" />
+   *           &lt;field column="type_s" />
+   *         &lt;/entity>
+   *         &lt;entity child="true" name="parentChild0Child1" query="select * from parentChild0Child1 where parentChild0Child1_parent_id='${parentChild0.id}'">
+   *           &lt;field column="id" />
+   *           &lt;field column="desc" />
+   *           &lt;field column="type_s" />
+   *         &lt;/entity>
+   *       &lt;/entity>
+   *       &lt;entity child="true" name="parentChild1" query="select * from parentChild1 where parentChild1_parent_id='${parent.id}'">
+   *         &lt;field column="id" />
+   *         &lt;field column="desc" />
+   *         &lt;field column="type_s" />
+   *         &lt;entity child="true" name="parentChild1Child0" query="select * from parentChild1Child0 where parentChild1Child0_parent_id='${parentChild1.id}'">
+   *           &lt;field column="id" />
+   *           &lt;field column="desc" />
+   *           &lt;field column="type_s" />
+   *         &lt;/entity>
+   *         &lt;entity child="true" name="parentChild1Child1" query="select * from parentChild1Child1 where parentChild1Child1_parent_id='${parentChild1.id}'">
+   *           &lt;field column="id" />
+   *           &lt;field column="desc" />
+   *           &lt;field column="type_s" />
+   *         &lt;/entity>
+   *       &lt;/entity>
+   *     &lt;/entity>
+   *   &lt;/document>
+   * &lt;/dataConfig>
+   * 
+   * </pre>
+   * 
+   * Internally configures MockDataSource.
+   **/
   private String createRandomizedConfig(int depth, String parentType, int parentsNum, ContextHolder holder) {
     List<Hierarchy> parentData = createMockedIterator(parentType, "SELECT * FROM " + parentType, parentsNum, holder);
     
-    // each map represents parent and each parent is root of separate hierarchy
     holder.hierarchies = parentData;
     
     String children = createChildren(parentType, 0, depth, parentData, holder);
@@ -293,7 +368,10 @@ public class TestHierarchicalDocBuilder
       Map<String, Object> data = parentHierarchy.elementData;
       String id = (String) data.get(FIELD_ID);
       String select = String.format(Locale.ROOT, "select * from %s where %s='%s'", type, type + "_parent_id", id);
-      List<Hierarchy> childHierarchies = createMockedIterator(type, select, atLeast(5), holder);
+      
+      // Number of actual children documents
+      int childrenNum = 1 + random().nextInt(3);
+      List<Hierarchy> childHierarchies = createMockedIterator(type, select, childrenNum, holder);
       parentHierarchy.elements.addAll(childHierarchies);
       result.addAll(childHierarchies);
     }
@@ -307,7 +385,9 @@ public class TestHierarchicalDocBuilder
       return "";
     }
     
-    int childrenNumber = atLeast(2);
+    // number of different children <b>types</b> of parent, i.e. parentChild0, parentChild1
+    // @see #createMockedIterator for the actual number of each children type 
+    int childrenNumber = 2 + random().nextInt(3);
     StringBuilder builder = new StringBuilder();
     for (int childIndex = 0; childIndex < childrenNumber; ++childIndex) {
       String childName = parentName + "Child" + childIndex;