You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nlpcraft.apache.org by se...@apache.org on 2022/03/24 08:55:53 UTC
[incubator-nlpcraft] branch master updated: JMH based performance tests improved.
This is an automated email from the ASF dual-hosted git repository.
sergeykamov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nlpcraft.git
The following commit(s) were added to refs/heads/master by this push:
new f8e3b8f JMH based performance tests improved.
f8e3b8f is described below
commit f8e3b8f07e4f09611a9337128612bcc714d5d48a
Author: Sergey Kamov <sk...@gmail.com>
AuthorDate: Thu Mar 24 11:55:42 2022 +0300
JMH based performance tests improved.
---
nlpcraft/src/test/resources/log4j2.xml | 2 +-
.../nlpcraft/nlp/benchmark/client/2022-03-24.json | 57 ++++++++++
.../nlp/benchmark/client/NCClientBenchmark.java | 121 +++++++++++++++++++++
.../opennlp/NCEnOpenNLPTokenParserBenchmark.java | 73 -------------
.../apache/nlpcraft/nlp/util/NCTestRequest.scala | 2 +-
5 files changed, 180 insertions(+), 75 deletions(-)
diff --git a/nlpcraft/src/test/resources/log4j2.xml b/nlpcraft/src/test/resources/log4j2.xml
index cdf681c..cfe76d9 100644
--- a/nlpcraft/src/test/resources/log4j2.xml
+++ b/nlpcraft/src/test/resources/log4j2.xml
@@ -36,7 +36,7 @@
<AppenderRef ref="stdout"/>
<AppenderRef ref="stderr"/>
</Root>
- <Logger name="org.apache.nlpcraft" level="INFO" additivity="false">
+ <Logger name="org.apache.nlpcraft" level="${sys:TEST_LOG_LEVEL:-INFO}" additivity="false">
<AppenderRef ref="stdout"/>
<AppenderRef ref="stderr"/>
</Logger>
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/client/2022-03-24.json b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/client/2022-03-24.json
new file mode 100644
index 0000000..104e8f8
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/client/2022-03-24.json
@@ -0,0 +1,57 @@
+[
+ {
+ "jmhVersion" : "1.34",
+ "benchmark" : "org.apache.nlpcraft.nlp.benchmark.client.NCClientBenchmark.testParse",
+ "mode" : "avgt",
+ "threads" : 1,
+ "forks" : 1,
+ "jvm" : "/Library/Java/JavaVirtualMachines/jdk-11.0.9.jdk/Contents/Home/bin/java",
+ "jvmArgs" : [
+ "-Xms2G",
+ "-Xmx2G"
+ ],
+ "jdkVersion" : "11.0.9",
+ "vmName" : "Java HotSpot(TM) 64-Bit Server VM",
+ "vmVersion" : "11.0.9+7-LTS",
+ "warmupIterations" : 5,
+ "warmupTime" : "10 s",
+ "warmupBatchSize" : 1,
+ "measurementIterations" : 5,
+ "measurementTime" : "5 s",
+ "measurementBatchSize" : 1,
+ "primaryMetric" : {
+ "score" : 25.068108681069525,
+ "scoreError" : 6.605110263308195,
+ "scoreConfidence" : [
+ 18.46299841776133,
+ 31.67321894437772
+ ],
+ "scorePercentiles" : {
+ "0.0" : 22.4460412735426,
+ "50.0" : 25.551017474489797,
+ "90.0" : 27.0597884,
+ "95.0" : 27.0597884,
+ "99.0" : 27.0597884,
+ "99.9" : 27.0597884,
+ "99.99" : 27.0597884,
+ "99.999" : 27.0597884,
+ "99.9999" : 27.0597884,
+ "100.0" : 27.0597884
+ },
+ "scoreUnit" : "ms/op",
+ "rawData" : [
+ [
+ 22.4460412735426,
+ 24.557350980392158,
+ 25.551017474489797,
+ 25.726345276923077,
+ 27.0597884
+ ]
+ ]
+ },
+ "secondaryMetrics" : {
+ }
+ }
+]
+
+
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/client/NCClientBenchmark.java b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/client/NCClientBenchmark.java
new file mode 100644
index 0000000..79f4efe
--- /dev/null
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/client/NCClientBenchmark.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nlpcraft.nlp.benchmark.client;
+
+import org.apache.nlpcraft.NCEntity;
+import org.apache.nlpcraft.NCIntent;
+import org.apache.nlpcraft.NCIntentTerm;
+import org.apache.nlpcraft.NCModelClient;
+import org.apache.nlpcraft.NCRequest;
+import org.apache.nlpcraft.NCResult;
+import org.apache.nlpcraft.NCResultType;
+import org.apache.nlpcraft.nlp.util.NCTestModelAdapter;
+import org.apache.nlpcraft.nlp.util.NCTestRequest;
+import org.apache.nlpcraft.nlp.util.NCTestUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.results.format.ResultFormatType;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Fork(value = 1, jvmArgs = {"-Xms2G", "-Xmx2G"})
+@Warmup(iterations = 5, time = 10)
+@Measurement(iterations = 5, time = 5)
+public class NCClientBenchmark {
+ // It overrides 'org.apache.nlpcraft' log level value in test scope log4j.xml property file.
+ // Should be here to be loaded before logger initialized.
+ static {
+ System.setProperty("TEST_LOG_LEVEL", "ERROR");
+ }
+
+ @State(Scope.Thread)
+ public static class NCBenchmarkAdapterState {
+ final NCRequest request = NCTestRequest.apply(
+ "I want to use JMH framework for test the system performance, so - lights on at second floor kitchen!"
+ );
+ }
+
+ private NCModelClient client;
+
+ @Setup
+ public void setUp() {
+ client = new NCModelClient(
+ new NCTestModelAdapter() {
+ private final NCResult res = new NCResult("OK", NCResultType.ASK_RESULT);
+
+ {
+ getPipeline().getEntityParsers().add(NCTestUtils.mkENSemanticParser("models/lightswitch_model.yaml"));
+ }
+
+ @NCIntent("intent=ls term(act)={# == 'ls:on'} term(loc)={# == 'ls:loc'}*")
+ public NCResult onMatch(@NCIntentTerm("act") NCEntity act, @NCIntentTerm("loc")List<NCEntity> locs) {
+ return res;
+ }
+ }
+ );
+ }
+
+ @TearDown
+ public void tearDown() {
+ if (client != null)
+ client.close();
+ }
+
+ @Benchmark
+ public void testParse(Blackhole bh, NCBenchmarkAdapterState state) {
+ bh.consume(client.ask(state.request.getText(), null, state.request.getUserId()));
+ }
+
+ /**
+ *
+ * @param args
+ * @throws RunnerException
+ */
+ public static void main(String[] args) throws RunnerException {
+ new Runner(new OptionsBuilder().
+ include(NCClientBenchmark.class.getSimpleName()).
+ shouldFailOnError(true).
+ resultFormat(ResultFormatType.JSON).
+ result("nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/client/" + DateTimeFormatter.ISO_LOCAL_DATE.format(LocalDate.now()) + ".json").
+ build()
+ ).run();
+ }
+}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/token/parser/opennlp/NCEnOpenNLPTokenParserBenchmark.java b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/token/parser/opennlp/NCEnOpenNLPTokenParserBenchmark.java
deleted file mode 100644
index 33581d1..0000000
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/benchmark/token/parser/opennlp/NCEnOpenNLPTokenParserBenchmark.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nlpcraft.nlp.benchmark.token.parser.opennlp;
-
-import org.apache.nlpcraft.NCRequest;
-import org.apache.nlpcraft.internal.util.NCResourceReader;
-import org.apache.nlpcraft.nlp.token.parser.NCOpenNLPTokenParser;
-import org.apache.nlpcraft.nlp.util.NCTestRequest;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
-import org.openjdk.jmh.runner.Runner;
-import org.openjdk.jmh.runner.RunnerException;
-import org.openjdk.jmh.runner.options.OptionsBuilder;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-@BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@State(Scope.Benchmark)
-@Fork(value = 1, jvmArgs = {"-Xms2G", "-Xmx2G"})
-@Warmup(iterations = 5, time = 10)
-@Measurement(iterations = 5, time = 5)
-public class NCEnOpenNLPTokenParserBenchmark {
- @State(Scope.Thread)
- public static class NCBenchmarkAdapterState {
- final NCRequest request = NCTestRequest.apply(
- "I am developing an integrated Benchmarking into an application, I want to use JMH as my framework."
- );
- }
-
- private NCOpenNLPTokenParser parser;
-
- @Setup
- public void setUp() {
- parser = new NCOpenNLPTokenParser(NCResourceReader.getPath("opennlp/en-token.bin"));
- }
-
- @Benchmark
- public void testParse(Blackhole bh, NCBenchmarkAdapterState state) {
- bh.consume(parser.tokenize(state.request.getText()));
- }
-
- public static void main(String[] args) throws RunnerException {
- new Runner(new OptionsBuilder().include(NCEnOpenNLPTokenParserBenchmark.class.getSimpleName()).shouldFailOnError(true).build()).run();
- }
-}
diff --git a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestRequest.scala b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestRequest.scala
index 89c81d9..c087b0b 100644
--- a/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestRequest.scala
+++ b/nlpcraft/src/test/scala/org/apache/nlpcraft/nlp/util/NCTestRequest.scala
@@ -35,7 +35,7 @@ import scala.jdk.CollectionConverters.*
*/
case class NCTestRequest(
txt: String,
- userId: String = null,
+ userId: String = "userId",
reqId: String = null,
ts: Long = -1,
data: Map[String, AnyRef] = null