You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2023/11/29 11:23:59 UTC

(lucene) branch branch_9x updated: Report the time it took for building the FST in Test2BFST (#12847)

This is an automated email from the ASF dual-hosted git repository.

mikemccand pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new 92a5e5b02e0 Report the time it took for building the FST in Test2BFST (#12847)
92a5e5b02e0 is described below

commit 92a5e5b02e0e083126c4122f2b7a02426c21a037
Author: Dzung Bui <du...@gmail.com>
AuthorDate: Wed Nov 29 20:17:54 2023 +0900

    Report the time it took for building the FST in Test2BFST (#12847)
    
    * Report the time it took for building the FST
    
    * Update CHANGES
    
    * Change ramBytesUsed to numBytes
    
    * Report the verification time
    
    * Rename to fstSizeInBytes
---
 lucene/CHANGES.txt                                 |  7 +++++
 .../org/apache/lucene/util/fst/FSTCompiler.java    |  4 +++
 .../test/org/apache/lucene/util/fst/Test2BFST.java | 32 +++++++++++++++-------
 3 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 314d8d383ce..63a95b77818 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -43,6 +43,9 @@ API Changes
 * GITHUB#12799: Make TaskExecutor constructor public and use TaskExecutor for concurrent
   HNSW graph build. (Shubham Chaudhary)
 
+* GITHUB#12758, GITHUB#12803: Remove FST constructor with DataInput for metadata. Please
+  use the constructor with FSTMetadata instead. (Anh Dung Bui)
+
 New Features
 ---------------------
 
@@ -111,6 +114,10 @@ Improvements
   minimal FST.  Inspired by this Rust FST implemention:
   https://blog.burntsushi.net/transducers (Mike McCandless)
 
+* GITHUB#12738: NodeHash now stores the FST nodes data instead of just node addresses (Anh Dung Bui)
+
+* GITHUB#12847: Test2BFST now reports the time it took to build the FST and the real FST size (Anh Dung Bui)
+
 Optimizations
 ---------------------
 * GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
index 7f9a5ec7958..be6bb663ca9 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -867,6 +867,10 @@ public class FSTCompiler<T> {
     return fst.ramBytesUsed();
   }
 
+  public long fstSizeInBytes() {
+    return bytes.getPosition();
+  }
+
   static final class CompiledNode implements Node {
     long node;
 
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
index 842e241dd72..2e0aefbc06d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -66,6 +66,7 @@ public class Test2BFST extends LuceneTestCase {
         Random r = new Random(seed);
         int[] ints2 = new int[200];
         IntsRef input2 = new IntsRef(ints2, 0, ints2.length);
+        long startTime = System.nanoTime();
         while (true) {
           // System.out.println("add: " + input + " -> " + output);
           for (int i = 10; i < ints2.length; i++) {
@@ -78,9 +79,13 @@ public class Test2BFST extends LuceneTestCase {
                 count
                     + ": "
                     + fstCompiler.fstRamBytesUsed()
-                    + " bytes; "
+                    + " RAM bytes used; "
+                    + fstCompiler.fstSizeInBytes()
+                    + " FST bytes; "
                     + fstCompiler.getNodeCount()
-                    + " nodes");
+                    + " nodes; took "
+                    + (long) ((System.nanoTime() - startTime) / 1e9)
+                    + " seconds");
           }
           if (fstCompiler.getNodeCount() > Integer.MAX_VALUE + 100L * 1024 * 1024) {
             break;
@@ -93,7 +98,7 @@ public class Test2BFST extends LuceneTestCase {
         for (int verify = 0; verify < 2; verify++) {
           System.out.println(
               "\nTEST: now verify [fst size="
-                  + fst.ramBytesUsed()
+                  + fst.numBytes()
                   + "; nodeCount="
                   + fstCompiler.getNodeCount()
                   + "; arcCount="
@@ -103,9 +108,11 @@ public class Test2BFST extends LuceneTestCase {
           Arrays.fill(ints2, 0);
           r = new Random(seed);
 
+          startTime = System.nanoTime();
           for (int i = 0; i < count; i++) {
             if (i % 1000000 == 0) {
-              System.out.println(i + "...: ");
+              System.out.println(
+                  i + "...: took " + (long) ((System.nanoTime() - startTime) / 1e9) + " seconds");
             }
             for (int j = 10; j < ints2.length; j++) {
               ints2[j] = r.nextInt(256);
@@ -168,7 +175,7 @@ public class Test2BFST extends LuceneTestCase {
           fstCompiler.add(input, BytesRef.deepCopyOf(output));
           count++;
           if (count % 10000 == 0) {
-            long size = fstCompiler.fstRamBytesUsed();
+            long size = fstCompiler.fstSizeInBytes();
             if (count % 1000000 == 0) {
               System.out.println(count + "...: " + size + " bytes");
             }
@@ -184,7 +191,7 @@ public class Test2BFST extends LuceneTestCase {
 
           System.out.println(
               "\nTEST: now verify [fst size="
-                  + fst.ramBytesUsed()
+                  + fst.numBytes()
                   + "; nodeCount="
                   + fstCompiler.getNodeCount()
                   + "; arcCount="
@@ -194,9 +201,12 @@ public class Test2BFST extends LuceneTestCase {
           r = new Random(seed);
           Arrays.fill(ints, 0);
 
+          long startTime = System.nanoTime();
+
           for (int i = 0; i < count; i++) {
             if (i % 1000000 == 0) {
-              System.out.println(i + "...: ");
+              System.out.println(
+                  i + "...: took " + (long) ((System.nanoTime() - startTime) / 1e9) + " seconds");
             }
             r.nextBytes(outputBytes);
             assertEquals(output, Util.get(fst, input));
@@ -255,7 +265,7 @@ public class Test2BFST extends LuceneTestCase {
           output += 1 + r.nextInt(10);
           count++;
           if (count % 10000 == 0) {
-            long size = fstCompiler.fstRamBytesUsed();
+            long size = fstCompiler.fstSizeInBytes();
             if (count % 1000000 == 0) {
               System.out.println(count + "...: " + size + " bytes");
             }
@@ -272,7 +282,7 @@ public class Test2BFST extends LuceneTestCase {
 
           System.out.println(
               "\nTEST: now verify [fst size="
-                  + fst.ramBytesUsed()
+                  + fst.numBytes()
                   + "; nodeCount="
                   + fstCompiler.getNodeCount()
                   + "; arcCount="
@@ -283,9 +293,11 @@ public class Test2BFST extends LuceneTestCase {
 
           output = 1;
           r = new Random(seed);
+          long startTime = System.nanoTime();
           for (int i = 0; i < count; i++) {
             if (i % 1000000 == 0) {
-              System.out.println(i + "...: ");
+              System.out.println(
+                  i + "...: took " + (long) ((System.nanoTime() - startTime) / 1e9) + " seconds");
             }
 
             assertEquals(output, Util.get(fst, input).longValue());