You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/10/11 06:18:55 UTC
(opennlp) 01/03: Add a JMH benchmark for SentenceVectorsDL
This is an automated email from the ASF dual-hosted git repository.
krickert pushed a commit to branch feat/embeddings-timings-tests
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 21a81370710f377a757f9ad3dd2bddd2aa61f166
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Sun Oct 11 01:53:43 2026 -0400
Add a JMH benchmark for SentenceVectorsDL
SentenceVectorsDLBenchmark times embed per text and embedAll for calls
of 1 to 128 texts of mixed length, on the CPU or CUDA device 0. The
opennlp-dl module gets the same jmh profile as opennlp-runtime.
Signed-off-by: Kristian Rickert <kr...@apache.org>
---
opennlp-core/opennlp-ml/opennlp-dl/pom.xml | 69 +++++++++-
.../dl/vectors/SentenceVectorsDLBenchmark.java | 146 +++++++++++++++++++++
2 files changed, 214 insertions(+), 1 deletion(-)
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/pom.xml b/opennlp-core/opennlp-ml/opennlp-dl/pom.xml
index 55d887ca3..5d994a4af 100644
--- a/opennlp-core/opennlp-ml/opennlp-dl/pom.xml
+++ b/opennlp-core/opennlp-ml/opennlp-dl/pom.xml
@@ -89,4 +89,71 @@
</plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+
+ <profiles>
+ <profile>
+ <id>jmh</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>${jmh.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>3.6.2</version>
+ <executions>
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/jmh/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>jmh-compile</id>
+ <phase>test-compile</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ <configuration>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>${jmh.version}</version>
+ </path>
+ </annotationProcessorPaths>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/jmh/java/opennlp/dl/vectors/SentenceVectorsDLBenchmark.java b/opennlp-core/opennlp-ml/opennlp-dl/src/jmh/java/opennlp/dl/vectors/SentenceVectorsDLBenchmark.java
new file mode 100644
index 000000000..30695cb1c
--- /dev/null
+++ b/opennlp-core/opennlp-ml/opennlp-dl/src/jmh/java/opennlp/dl/vectors/SentenceVectorsDLBenchmark.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.dl.vectors;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import opennlp.dl.InferenceOptions;
+
+/**
+ * Times {@link SentenceVectorsDL} on a real sentence-transformers model, one call of
+ * {@code callSize} texts per operation, on the CPU or a CUDA device.
+ *
+ * <p>The texts are 4 to 40 words drawn with a fixed seed from a small word list, so a call
+ * mixes tokenized lengths the way natural text does. Run with {@code -prof gc} to see the
+ * bytes allocated per call.</p>
+ *
+ * <p>The model directory holds {@code model.onnx} and {@code vocab.txt}, for example
+ * {@code all-MiniLM-L6-v2}. It defaults to {@code ~/.opennlp/onnx/sentence-transformers} and is
+ * set with {@code -p modelDir=<dir>}.</p>
+ */
+@State(Scope.Benchmark)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Warmup(iterations = 3, time = 2)
+@Measurement(iterations = 5, time = 2)
+@Fork(1)
+public class SentenceVectorsDLBenchmark {
+
+ private static final String[] WORDS = {"the", "model", "reads", "a", "short", "sentence",
+ "and", "returns", "one", "vector", "for", "search", "over", "news", "articles", "with",
+ "many", "different", "lengths", "while", "batching", "keeps", "throughput", "high",
+ "on", "both", "devices", "today", "tomorrow", "city", "council", "approved", "budget"};
+
+ private static final int MIN_WORDS = 4;
+
+ private static final int MAX_WORDS = 40;
+
+ private static final long SEED = 42;
+
+ /** The directory with {@code model.onnx} and {@code vocab.txt}; empty for the default. */
+ @Param({""})
+ public String modelDir;
+
+ /** {@code cpu} or {@code gpu} (CUDA device 0). */
+ @Param({"cpu"})
+ public String device;
+
+ /** The number of texts in one call. */
+ @Param({"1", "8", "32", "128"})
+ public int callSize;
+
+ private SentenceVectorsDL vectors;
+
+ private List<String> texts;
+
+ /**
+ * Opens the session and builds the texts of one call.
+ *
+ * @throws Exception Thrown if the model cannot be loaded or the device is not usable.
+ */
+ @Setup(Level.Trial)
+ public void setUp() throws Exception {
+ final File dir = modelDir.isEmpty()
+ ? new File(System.getProperty("user.home"), ".opennlp/onnx/sentence-transformers")
+ : new File(modelDir);
+ final InferenceOptions options = new InferenceOptions();
+ options.setGpu("gpu".equals(device));
+ vectors = new SentenceVectorsDL(new File(dir, "model.onnx"), new File(dir, "vocab.txt"),
+ true, Pooling.MEAN, true, SentenceVectorsDL.DEFAULT_MAX_LENGTH, options);
+ final Random random = new Random(SEED);
+ texts = new ArrayList<>(callSize);
+ for (int i = 0; i < callSize; i++) {
+ final int words = MIN_WORDS + random.nextInt(MAX_WORDS - MIN_WORDS + 1);
+ final StringBuilder text = new StringBuilder();
+ for (int w = 0; w < words; w++) {
+ if (w > 0) {
+ text.append(' ');
+ }
+ text.append(WORDS[random.nextInt(WORDS.length)]);
+ }
+ texts.add(text.toString());
+ }
+ }
+
+ /** Closes the session. */
+ @TearDown(Level.Trial)
+ public void tearDown() {
+ vectors.close();
+ }
+
+ /**
+ * Embeds the call one text at a time, as a caller without {@code embedAll} would.
+ *
+ * @param blackhole Consumes the vectors.
+ */
+ @Benchmark
+ public void embedEach(final Blackhole blackhole) {
+ for (final String text : texts) {
+ blackhole.consume(vectors.embed(text));
+ }
+ }
+
+ /**
+ * Embeds the call with one {@code embedAll}.
+ *
+ * @return The vectors.
+ */
+ @Benchmark
+ public float[][] embedAll() {
+ return vectors.embedAll(texts);
+ }
+}