You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2023/11/29 11:23:59 UTC
(lucene) branch branch_9x updated: Report the time it took for building the FST in Test2BFST (#12847)
This is an automated email from the ASF dual-hosted git repository.
mikemccand pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 92a5e5b02e0 Report the time it took for building the FST in Test2BFST (#12847)
92a5e5b02e0 is described below
commit 92a5e5b02e0e083126c4122f2b7a02426c21a037
Author: Dzung Bui <du...@gmail.com>
AuthorDate: Wed Nov 29 20:17:54 2023 +0900
Report the time it took for building the FST in Test2BFST (#12847)
* Report the time it took for building the FST
* Update CHANGES
* Change ramBytesUsed to numBytes
* Report the verification time
* Rename to fstSizeInBytes
---
lucene/CHANGES.txt | 7 +++++
.../org/apache/lucene/util/fst/FSTCompiler.java | 4 +++
.../test/org/apache/lucene/util/fst/Test2BFST.java | 32 +++++++++++++++-------
3 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 314d8d383ce..63a95b77818 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -43,6 +43,9 @@ API Changes
* GITHUB#12799: Make TaskExecutor constructor public and use TaskExecutor for concurrent
HNSW graph build. (Shubham Chaudhary)
+* GITHUB#12758, GITHUB#12803: Remove FST constructor with DataInput for metadata. Please
+ use the constructor with FSTMetadata instead. (Anh Dung Bui)
+
New Features
---------------------
@@ -111,6 +114,10 @@ Improvements
minimal FST. Inspired by this Rust FST implemention:
https://blog.burntsushi.net/transducers (Mike McCandless)
+* GITHUB#12738: NodeHash now stores the FST nodes data instead of just node addresses (Anh Dung Bui)
+
+* GITHUB#12847: Test2BFST now reports the time it took to build the FST and the real FST size (Anh Dung Bui)
+
Optimizations
---------------------
* GITHUB#12183: Make TermStates#build concurrent. (Shubham Chaudhary)
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
index 7f9a5ec7958..be6bb663ca9 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -867,6 +867,10 @@ public class FSTCompiler<T> {
return fst.ramBytesUsed();
}
+ public long fstSizeInBytes() {
+ return bytes.getPosition();
+ }
+
static final class CompiledNode implements Node {
long node;
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
index 842e241dd72..2e0aefbc06d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -66,6 +66,7 @@ public class Test2BFST extends LuceneTestCase {
Random r = new Random(seed);
int[] ints2 = new int[200];
IntsRef input2 = new IntsRef(ints2, 0, ints2.length);
+ long startTime = System.nanoTime();
while (true) {
// System.out.println("add: " + input + " -> " + output);
for (int i = 10; i < ints2.length; i++) {
@@ -78,9 +79,13 @@ public class Test2BFST extends LuceneTestCase {
count
+ ": "
+ fstCompiler.fstRamBytesUsed()
- + " bytes; "
+ + " RAM bytes used; "
+ + fstCompiler.fstSizeInBytes()
+ + " FST bytes; "
+ fstCompiler.getNodeCount()
- + " nodes");
+ + " nodes; took "
+ + (long) ((System.nanoTime() - startTime) / 1e9)
+ + " seconds");
}
if (fstCompiler.getNodeCount() > Integer.MAX_VALUE + 100L * 1024 * 1024) {
break;
@@ -93,7 +98,7 @@ public class Test2BFST extends LuceneTestCase {
for (int verify = 0; verify < 2; verify++) {
System.out.println(
"\nTEST: now verify [fst size="
- + fst.ramBytesUsed()
+ + fst.numBytes()
+ "; nodeCount="
+ fstCompiler.getNodeCount()
+ "; arcCount="
@@ -103,9 +108,11 @@ public class Test2BFST extends LuceneTestCase {
Arrays.fill(ints2, 0);
r = new Random(seed);
+ startTime = System.nanoTime();
for (int i = 0; i < count; i++) {
if (i % 1000000 == 0) {
- System.out.println(i + "...: ");
+ System.out.println(
+ i + "...: took " + (long) ((System.nanoTime() - startTime) / 1e9) + " seconds");
}
for (int j = 10; j < ints2.length; j++) {
ints2[j] = r.nextInt(256);
@@ -168,7 +175,7 @@ public class Test2BFST extends LuceneTestCase {
fstCompiler.add(input, BytesRef.deepCopyOf(output));
count++;
if (count % 10000 == 0) {
- long size = fstCompiler.fstRamBytesUsed();
+ long size = fstCompiler.fstSizeInBytes();
if (count % 1000000 == 0) {
System.out.println(count + "...: " + size + " bytes");
}
@@ -184,7 +191,7 @@ public class Test2BFST extends LuceneTestCase {
System.out.println(
"\nTEST: now verify [fst size="
- + fst.ramBytesUsed()
+ + fst.numBytes()
+ "; nodeCount="
+ fstCompiler.getNodeCount()
+ "; arcCount="
@@ -194,9 +201,12 @@ public class Test2BFST extends LuceneTestCase {
r = new Random(seed);
Arrays.fill(ints, 0);
+ long startTime = System.nanoTime();
+
for (int i = 0; i < count; i++) {
if (i % 1000000 == 0) {
- System.out.println(i + "...: ");
+ System.out.println(
+ i + "...: took " + (long) ((System.nanoTime() - startTime) / 1e9) + " seconds");
}
r.nextBytes(outputBytes);
assertEquals(output, Util.get(fst, input));
@@ -255,7 +265,7 @@ public class Test2BFST extends LuceneTestCase {
output += 1 + r.nextInt(10);
count++;
if (count % 10000 == 0) {
- long size = fstCompiler.fstRamBytesUsed();
+ long size = fstCompiler.fstSizeInBytes();
if (count % 1000000 == 0) {
System.out.println(count + "...: " + size + " bytes");
}
@@ -272,7 +282,7 @@ public class Test2BFST extends LuceneTestCase {
System.out.println(
"\nTEST: now verify [fst size="
- + fst.ramBytesUsed()
+ + fst.numBytes()
+ "; nodeCount="
+ fstCompiler.getNodeCount()
+ "; arcCount="
@@ -283,9 +293,11 @@ public class Test2BFST extends LuceneTestCase {
output = 1;
r = new Random(seed);
+ long startTime = System.nanoTime();
for (int i = 0; i < count; i++) {
if (i % 1000000 == 0) {
- System.out.println(i + "...: ");
+ System.out.println(
+ i + "...: took " + (long) ((System.nanoTime() - startTime) / 1e9) + " seconds");
}
assertEquals(output, Util.get(fst, input).longValue());