[lucene-solr] branch master updated: LUCENE-9760: Hunspell: print total memory usage in TestAllDictionaries, cleanup (#2347)

[dw...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/master by this push:
     new c0d3244  LUCENE-9760: Hunspell: print total memory usage in TestAllDictionaries, cleanup (#2347)
c0d3244 is described below

commit c0d32444be4d1b2fbd4fc1d282cee9b223df2610
Author: Peter Gromov <pe...@jetbrains.com>
AuthorDate: Thu Feb 11 09:11:09 2021 +0100

    LUCENE-9760: Hunspell: print total memory usage in TestAllDictionaries, cleanup (#2347)
---
 .../analysis/hunspell/TestAllDictionaries.java     | 52 +++++++++++++---------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
index 308fa79..f64c6d8 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
@@ -37,6 +37,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -44,6 +45,7 @@ import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.apache.lucene.util.NamedThreadFactory;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.RamUsageTester;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -147,28 +149,33 @@ public class TestAllDictionaries extends LuceneTestCase {
   }
 
   public void testDictionariesLoadSuccessfully() throws Exception {
+    AtomicLong totalMemory = new AtomicLong();
+    AtomicLong totalWords = new AtomicLong();
     int threads = Runtime.getRuntime().availableProcessors();
     ExecutorService executor =
         Executors.newFixedThreadPool(threads, new NamedThreadFactory("dictCheck-"));
+    List<Path> failures = Collections.synchronizedList(new ArrayList<>());
+    Function<Path, Void> process =
+        (Path aff) -> {
+          try {
+            Dictionary dic = loadDictionary(aff);
+            totalMemory.addAndGet(RamUsageTester.sizeOf(dic));
+            totalWords.addAndGet(RamUsageTester.sizeOf(dic.words));
+            System.out.println(aff + "\t" + memoryUsageSummary(dic));
+          } catch (Throwable e) {
+            failures.add(aff);
+            System.err.println("While checking " + aff + ":");
+            e.printStackTrace();
+          }
+          return null;
+        };
+
+    List<Callable<Void>> tasks =
+        findAllAffixFiles()
+            .map(aff -> (Callable<Void>) () -> process.apply(aff))
+            .collect(Collectors.toList());
     try {
-      List<Path> failures = Collections.synchronizedList(new ArrayList<>());
-      Function<Path, Void> process =
-          (Path aff) -> {
-            try {
-              System.out.println(aff + "\t" + memoryUsage(loadDictionary(aff)));
-            } catch (Throwable e) {
-              failures.add(aff);
-              System.err.println("While checking " + aff + ":");
-              e.printStackTrace();
-            }
-            return null;
-          };
-
-      for (Future<?> future :
-          executor.invokeAll(
-              findAllAffixFiles()
-                  .map(aff -> (Callable<?>) () -> process.apply(aff))
-                  .collect(Collectors.toList()))) {
+      for (Future<?> future : executor.invokeAll(tasks)) {
         future.get();
       }
 
@@ -181,11 +188,16 @@ public class TestAllDictionaries extends LuceneTestCase {
       }
     } finally {
       executor.shutdown();
-      executor.awaitTermination(1, TimeUnit.MINUTES);
+      assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
     }
+
+    System.out.println("Total dictionaries loaded: " + tasks.size());
+    System.out.println("Total memory: " + RamUsageEstimator.humanReadableUnits(totalMemory.get()));
+    System.out.println(
+        "Total memory for word storage: " + RamUsageEstimator.humanReadableUnits(totalWords.get()));
   }
 
-  private static String memoryUsage(Dictionary dic) {
+  private static String memoryUsageSummary(Dictionary dic) {
     return RamUsageTester.humanSizeOf(dic)
         + "\t("
         + ("words=" + RamUsageTester.humanSizeOf(dic.words) + ", ")