You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2021/06/24 14:13:21 UTC

[lucene-solr] branch branch_8x updated: Enhance Lucene benchmark module's ReutersContentSource to use more concurrency while indexing

This is an automated email from the ASF dual-hosted git repository.

mikemccand pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/branch_8x by this push:
     new e09979c  Enhance Lucene benchmark module's ReutersContentSource to use more concurrency while indexing
e09979c is described below

commit e09979c8328ad5857cb5484ffb8f62128a2bdaf3
Author: Mike McCandless <mi...@apache.org>
AuthorDate: Thu Jun 24 10:12:49 2021 -0400

    Enhance Lucene benchmark module's ReutersContentSource to use more concurrency while indexing
---
 .../conf/indexing-flush-by-RAM-multithreaded.alg   |  2 +-
 .../org/apache/lucene/benchmark/Constants.java     |  2 +
 .../byTask/feeds/ReutersContentSource.java         | 54 +++++++++++++++-------
 .../benchmark/byTask/tasks/TaskSequence.java       | 16 ++++++-
 .../lucene/benchmark/byTask/utils/Config.java      |  9 ++++
 5 files changed, 63 insertions(+), 20 deletions(-)

diff --git a/lucene/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg b/lucene/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg
index 4f73259..84eac11 100644
--- a/lucene/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg
+++ b/lucene/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg
@@ -53,7 +53,7 @@ log.queries=true
 
     { "Populate"
         CreateIndex
-        [{ "MAddDocs" AddDoc } : 5000] : 4
+        [{ {{"MAddDocs" AddDoc } : 5000} FlushIndex } ] : 8
         ForceMerge(1)
         CloseIndex
     }
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/Constants.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/Constants.java
index b03cdcd..67fc57b 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/Constants.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/Constants.java
@@ -28,4 +28,6 @@ public class Constants {
     public static Boolean[] BOOLEANS = new Boolean[] { Boolean.FALSE, Boolean.TRUE };
 
     public static final int DEFAULT_MAXIMUM_DOCUMENTS = Integer.MAX_VALUE;
+
+    public static final String PARALLEL_TASK_THREAD_NAME_PREFIX = "ParallelTaskThread";
 }
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersContentSource.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersContentSource.java
index 64d5e6e..efcea6b 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersContentSource.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersContentSource.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.Locale;
 
+import org.apache.lucene.benchmark.Constants;
 import org.apache.lucene.benchmark.byTask.utils.Config;
 
 /**
@@ -52,8 +53,8 @@ public class ReutersContentSource extends ContentSource {
   private ThreadLocal<DateFormatInfo> dateFormat = new ThreadLocal<>();
   private Path dataDir = null;
   private ArrayList<Path> inputFiles = new ArrayList<>();
-  private int nextFile = 0;
-  private int iteration = 0;
+  private int[] docCountArr;
+  private volatile boolean docCountArrCreated;
   
   @Override
   public void setConfig(Config config) {
@@ -103,21 +104,35 @@ public class ReutersContentSource extends ContentSource {
   
   @Override
   public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException {
-    Path f = null;
-    String name = null;
-    synchronized (this) {
-      if (nextFile >= inputFiles.size()) {
-        // exhausted files, start a new round, unless forever set to false.
-        if (!forever) {
-          throw new NoMoreDataException();
-        }
-        nextFile = 0;
-        iteration++;
-      }
-      f = inputFiles.get(nextFile++);
-      name = f.toRealPath() + "_" + iteration;
+    if (docCountArrCreated == false) {
+      docCountArrInit();
     }
 
+    int threadIndexSize = Thread.currentThread().getName().length();
+    int parallelTaskThreadSize = Constants.PARALLEL_TASK_THREAD_NAME_PREFIX.length();
+
+    // Extract ThreadIndex from unique ThreadName which is set with '"ParallelTaskThread-"+index',
+    // in TaskSequence.java's doParallelTasks()
+    int threadIndex =
+        Integer.parseInt(
+            Thread.currentThread()
+                .getName()
+                .substring(parallelTaskThreadSize + 1, threadIndexSize));
+
+    assert (threadIndex >= 0 && threadIndex < docCountArr.length)
+        : "Please check threadIndex or docCountArr length";
+    int stride = threadIndex + docCountArr[threadIndex] * docCountArr.length;
+    int inFileSize = inputFiles.size();
+
+    // Modulo Operator covers all three possible senarios i.e. 1. If inputFiles.size() < Num Of
+    // Threads 2.inputFiles.size() == Num Of Threads 3.inputFiles.size() > Num Of Threads
+    int fileIndex = stride % inFileSize;
+    int iteration = stride / inFileSize;
+    docCountArr[threadIndex]++;
+
+    Path f = inputFiles.get(fileIndex);
+    String name = f.toRealPath() + "_" + iteration;
+
     try (BufferedReader reader = Files.newBufferedReader(f, StandardCharsets.UTF_8)) {
       // First line is the date, 3rd is the title, rest is body
       String dateStr = reader.readLine();
@@ -146,8 +161,13 @@ public class ReutersContentSource extends ContentSource {
   @Override
   public synchronized void resetInputs() throws IOException {
     super.resetInputs();
-    nextFile = 0;
-    iteration = 0;
+  }
+
+  private synchronized void docCountArrInit() {
+    if (docCountArrCreated == false) {
+      docCountArr = new int[getConfig().getNumThreads()];
+      docCountArrCreated = true;
+    }
   }
 
 }
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
index 4b2f94c..686242e 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/TaskSequence.java
@@ -17,11 +17,12 @@
 package org.apache.lucene.benchmark.byTask.tasks;
 
 
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
-import java.text.NumberFormat;
 
+import org.apache.lucene.benchmark.Constants;
 import org.apache.lucene.benchmark.byTask.PerfRunData;
 import org.apache.lucene.benchmark.byTask.feeds.NoMoreDataException;
 import org.apache.lucene.benchmark.byTask.stats.TaskStats;
@@ -345,12 +346,23 @@ public class TaskSequence extends PerfTask {
 
     initTasksArray();
     ParallelTask t[] = runningParallelTasks = new ParallelTask[repetitions * tasks.size()];
+    // Get number of parallel threads from algo file and set it to use in ReuersContentSource.java's
+    // docCountArrInit()
+    this.getRunData().getConfig().setNumThreads(t.length);
     // prepare threads
     int index = 0;
     for (int k=0; k<repetitions; k++) {
       for (int i = 0; i < tasksArray.length; i++) {
         final PerfTask task = tasksArray[i].clone();
-        t[index++] = new ParallelTask(task);
+        t[index] = new ParallelTask(task);
+        // Setting unique ThreadName with index value which is used in ReuersContentSource.java's
+        // getNextDocData().Please make changes
+        // in ReuersContentSource.java's getNextDocData() for
+        // Integer.parseInt(Thread.currentThread().getName().substring(parallelTaskThreadSize + 1,
+        // threadIndexSize))
+        // before making any modifications here
+        t[index].setName(Constants.PARALLEL_TASK_THREAD_NAME_PREFIX + "-" + index);
+        index++;
       }
     }
     // run threads
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java
index 1c7a98d..e0f3409 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java
@@ -54,6 +54,7 @@ public class Config {
   private HashMap<String, Object> valByRound = new HashMap<>();
   private HashMap<String, String> colForValByRound = new HashMap<>();
   private String algorithmText;
+  private int numThreads = 1;
 
   /**
    * Read both algorithm and config properties.
@@ -112,6 +113,14 @@ public class Config {
     }
   }
 
+  public void setNumThreads(int numThreads) {
+    this.numThreads = numThreads;
+  }
+
+  public int getNumThreads() {
+    return numThreads;
+  }
+
   @SuppressWarnings({"unchecked", "rawtypes"})
   private void printProps() {
     System.out.println("------------> config properties:");