You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by yo...@apache.org on 2021/12/23 14:03:47 UTC
[pulsar] 02/14: [pulsar-perf] Write histogram files for consume command (#12569)
This is an automated email from the ASF dual-hosted git repository.
yong pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit 42bce8a6e4f8eb3bcebdf720306932e2646401e3
Author: Matt Fleming <ma...@codeblueprint.co.uk>
AuthorDate: Fri Nov 5 09:36:44 2021 +0000
[pulsar-perf] Write histogram files for consume command (#12569)
* [pulsar-perf] Write histogram files for consume command
* [pulsar-perf] Disable writing to histogram files by default
Most users don't use the histogram files and instead opt for sending
metrics to prometheus, etc, so there's no need to have this enabled by
default.
Instead, add a new --histogram-file parameter to pulsar-perf
produce/consume which, when specified, dumps the contents of the
internal histogram to the given filename.
Previous behaviour can be achieved with the following options:
$ pulsar-perf produce --histogram-file perf-producer-$(date +%s).hgrm
* [pulsar-perf] Update docs with --histogram-file param
(cherry picked from commit 48de2e251ee3ea38449d696402d0cc57c88ee3c7)
---
.../pulsar/testclient/PerformanceConsumer.java | 24 +++++++++++++++++++--
.../pulsar/testclient/PerformanceProducer.java | 25 +++++++++++++++-------
2 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
index e0fa99b..5b1b863 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java
@@ -26,8 +26,8 @@ import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.beust.jcommander.Parameters;
import java.io.FileInputStream;
-import java.lang.management.BufferPoolMXBean;
-import java.lang.management.ManagementFactory;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.text.DecimalFormat;
import java.util.Collections;
@@ -38,6 +38,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.LongAdder;
import org.HdrHistogram.Histogram;
+import org.HdrHistogram.HistogramLogWriter;
import org.HdrHistogram.Recorder;
import org.apache.pulsar.client.api.ClientBuilder;
import org.apache.pulsar.client.api.Consumer;
@@ -185,6 +186,9 @@ public class PerformanceConsumer {
@Parameter(names = {"-bw", "--busy-wait"}, description = "Enable Busy-Wait on the Pulsar client")
public boolean enableBusyWait = false;
+
+ @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file")
+ public String histogramFile = null;
}
public static void main(String[] args) throws Exception {
@@ -400,7 +404,19 @@ public class PerformanceConsumer {
long oldTime = System.nanoTime();
Histogram reportHistogram = null;
+ HistogramLogWriter histogramLogWriter = null;
+
+ if (arguments.histogramFile != null) {
+ String statsFileName = arguments.histogramFile;
+ log.info("Dumping latency stats to {}", statsFileName);
+
+ PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
+ histogramLogWriter = new HistogramLogWriter(histogramLog);
+ // Some log header bits
+ histogramLogWriter.outputLogFormatVersion();
+ histogramLogWriter.outputLegend();
+ }
while (true) {
try {
@@ -425,6 +441,10 @@ public class PerformanceConsumer {
reportHistogram.getValueAtPercentile(99), reportHistogram.getValueAtPercentile(99.9),
reportHistogram.getValueAtPercentile(99.99), reportHistogram.getMaxValue());
+ if (histogramLogWriter != null) {
+ histogramLogWriter.outputIntervalHistogram(reportHistogram);
+ }
+
reportHistogram.reset();
oldTime = now;
}
diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
index f198a2d..c2d13c3 100644
--- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
+++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java
@@ -250,6 +250,9 @@ public class PerformanceProducer {
@Parameter(names = {"-fc", "--format-class"}, description="Custom Formatter class name")
public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter";
+
+ @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file")
+ public String histogramFile = null;
}
public static void main(String[] args) throws Exception {
@@ -429,16 +432,19 @@ public class PerformanceProducer {
long oldTime = System.nanoTime();
Histogram reportHistogram = null;
+ HistogramLogWriter histogramLogWriter = null;
- String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm";
- log.info("Dumping latency stats to {}", statsFileName);
+ if (arguments.histogramFile != null) {
+ String statsFileName = arguments.histogramFile;
+ log.info("Dumping latency stats to {}", statsFileName);
- PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
- HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog);
+ PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false);
+ histogramLogWriter = new HistogramLogWriter(histogramLog);
- // Some log header bits
- histogramLogWriter.outputLogFormatVersion();
- histogramLogWriter.outputLegend();
+ // Some log header bits
+ histogramLogWriter.outputLogFormatVersion();
+ histogramLogWriter.outputLegend();
+ }
while (true) {
try {
@@ -473,7 +479,10 @@ public class PerformanceProducer {
dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0),
dec.format(reportHistogram.getMaxValue() / 1000.0));
- histogramLogWriter.outputIntervalHistogram(reportHistogram);
+ if (histogramLogWriter != null) {
+ histogramLogWriter.outputIntervalHistogram(reportHistogram);
+ }
+
reportHistogram.reset();
oldTime = now;