[GitHub] carbondata pull request #2275: [CARBONDATA-2494] Fix lucene datasize and per...

[jackylk <gi...@git.apache.org>]

Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/2275#discussion_r189419746
  
    --- Diff: datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java ---
    @@ -212,69 +228,58 @@ private int getMaxDoc(Expression expression) {
           LOGGER.error(errorMessage);
           return null;
         }
    -
    -    // execute index search
    -    // initialize to null, else ScoreDoc objects will get accumulated in memory
    -    TopDocs result = null;
    -    try {
    -      result = indexSearcher.search(query, maxDocs);
    -    } catch (IOException e) {
    -      String errorMessage =
    -          String.format("failed to search lucene data, detail is %s", e.getMessage());
    -      LOGGER.error(errorMessage);
    -      throw new IOException(errorMessage);
    -    }
    -
         // temporary data, delete duplicated data
         // Map<BlockId, Map<BlockletId, Map<PageId, Set<RowId>>>>
    -    Map<String, Map<Integer, Set<Integer>>> mapBlocks = new HashMap<>();
    -
    -    for (ScoreDoc scoreDoc : result.scoreDocs) {
    -      // get a document
    -      Document doc = indexSearcher.doc(scoreDoc.doc);
    +    Map<String, Map<Integer, List<Short>>> mapBlocks = new HashMap<>();
    +
    +    for (Map.Entry<String, IndexSearcher> searcherEntry : indexSearcherMap.entrySet()) {
    +      IndexSearcher indexSearcher = searcherEntry.getValue();
    +      // execute index search
    +      // initialize to null, else ScoreDoc objects will get accumulated in memory
    +      TopDocs result = null;
    +      try {
    +        result = indexSearcher.search(query, maxDocs);
    +      } catch (IOException e) {
    +        String errorMessage =
    +            String.format("failed to search lucene data, detail is %s", e.getMessage());
    +        LOGGER.error(errorMessage);
    +        throw new IOException(errorMessage);
    +      }
     
    -      // get all fields
    -      List<IndexableField> fieldsInDoc = doc.getFields();
    +      ByteBuffer intBuffer = ByteBuffer.allocate(4);
     
    -      // get the blocklet id Map<BlockletId, Map<PageId, Set<RowId>>>
    -      String blockletId = fieldsInDoc.get(BLOCKLETID_ID).stringValue();
    -      Map<Integer, Set<Integer>> mapPageIds = mapBlocks.get(blockletId);
    -      if (mapPageIds == null) {
    -        mapPageIds = new HashMap<>();
    -        mapBlocks.put(blockletId, mapPageIds);
    -      }
    +      for (ScoreDoc scoreDoc : result.scoreDocs) {
    +        // get a document
    +        Document doc = indexSearcher.doc(scoreDoc.doc);
     
    -      // get the page id Map<PageId, Set<RowId>>
    -      Number pageId = fieldsInDoc.get(PAGEID_ID).numericValue();
    -      Set<Integer> setRowId = mapPageIds.get(pageId.intValue());
    -      if (setRowId == null) {
    -        setRowId = new HashSet<>();
    -        mapPageIds.put(pageId.intValue(), setRowId);
    +        // get all fields
    +        List<IndexableField> fieldsInDoc = doc.getFields();
    +        if (writeCacheSize > 0) {
    +          fillMap(intBuffer, mapBlocks, fieldsInDoc, searcherEntry.getKey());
    --- End diff --
    
    please add comment here to describe fillMap and fillMapDirect


---