You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2020/08/07 01:32:16 UTC
[kudu] branch master updated (40289e2 -> 7e50695)
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git.
from 40289e2 [KUDU-3177] Added kudu.snapshotTimestampMicros to kudu spark readOptions as optional property
new 433c177 [java] add state in ScanRequest toString
new f3c7b3f [scripts] alternative y-label in jobs_runtime.R
new 7e50695 [scripts] fix y-axis label in benchmarks.sh
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kudu/client/AsyncKuduScanner.java | 3 +-
src/kudu/scripts/benchmarks.sh | 57 ++++++++++++----------
src/kudu/scripts/jobs_runtime.R | 14 ++++--
3 files changed, 45 insertions(+), 29 deletions(-)
[kudu] 03/03: [scripts] fix y-axis label in benchmarks.sh
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 7e5069525a807dd64ce43b7c978739ccc642be0f
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Wed Aug 5 00:05:31 2020 -0700
[scripts] fix y-axis label in benchmarks.sh
This patch brings Y axis labels in sync with the actual metrics shown
in graphs generated by benchmarks.sh.
In addition, I split the graphs for the multi-thread insert test into
separate throughput and CPU efficiency graphs.
Change-Id: I2e5fb5efa742f2321ad8d52a794311defff620e9
Reviewed-on: http://gerrit.cloudera.org:8080/16293
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
---
src/kudu/scripts/benchmarks.sh | 57 ++++++++++++++++++++++++------------------
1 file changed, 32 insertions(+), 25 deletions(-)
diff --git a/src/kudu/scripts/benchmarks.sh b/src/kudu/scripts/benchmarks.sh
index b027e41..6154c7d 100755
--- a/src/kudu/scripts/benchmarks.sh
+++ b/src/kudu/scripts/benchmarks.sh
@@ -138,11 +138,12 @@ load_stats() {
}
write_img_plot() {
- local INPUT_FILE=$1
- local TEST_NAME=$2
+ local input_file="$1"
+ local test_name="$2"
+ local y_axis_label="$3"
# Rscript fails when there's only a header, so just skip
- if [ `wc -l $INPUT_FILE | cut -d ' ' -f1` -gt 1 ]; then
- Rscript jobs_runtime.R $INPUT_FILE $TEST_NAME
+ if [ `wc -l $input_file | cut -d ' ' -f1` -gt 1 ]; then
+ Rscript jobs_runtime.R $input_file $test_name "$y_axis_label"
fi
}
@@ -473,9 +474,9 @@ parse_and_record_all_results() {
record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_${metric}_latency $i $val
done
rate=$(grep -o 'Throughput.*' $log | awk '{print $2}')
- record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_throughput_wall $i $rate
+ record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_throughput $i $rate
rate=$(grep -o 'CPU efficiency.*' $log | awk '{print $3}')
- record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_throughput_cpu $i $rate
+ record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_cpu_efficiency $i $rate
done
# parse scan timings for scans and inserts with MRS only
@@ -622,10 +623,15 @@ generate_ycsb_plots() {
}
load_and_generate_plot() {
- local TEST_NAME=$1
- local PLOT_NAME=$2
- load_stats "$TEST_NAME" > $OUTDIR/$PLOT_NAME.tsv
- write_img_plot $OUTDIR/$PLOT_NAME.tsv $PLOT_NAME
+ local test_name=$1
+ local plot_name=$2
+ local y_axis_label=$3
+ if [ -z "$y_axis_label" ]; then
+ local y_axis_label="runtime (sec)"
+ fi
+ local fpath=${OUTDIR}/${plot_name}.tsv
+ load_stats "$test_name" > $fpath
+ write_img_plot $fpath $plot_name "$y_axis_label"
}
load_stats_and_generate_plots() {
@@ -647,15 +653,16 @@ load_stats_and_generate_plots() {
load_and_generate_plot "$BLOOM_TEST" bloom-test
load_and_generate_plot "$MT_BLOOM_TEST" mt-bloom-test
- load_and_generate_plot "${WIRE_PROTOCOL_TEST}" wire-protocol-test
- load_and_generate_plot "${WIRE_PROTOCOL_TEST}_columnar" wire-protocol-test-columnar
+ load_and_generate_plot "${WIRE_PROTOCOL_TEST}" wire-protocol-test "conversion rate (cycle/cell)"
+ load_and_generate_plot "${WIRE_PROTOCOL_TEST}_columnar" wire-protocol-test-columnar "conversion rate (cycle/cell)"
- load_and_generate_plot "$RPC_BENCH_TEST" rpc-bench-test
+ load_and_generate_plot "$RPC_BENCH_TEST" rpc-bench-test "request rate (RPC/sec)"
- load_and_generate_plot "${TS_INSERT_LATENCY}%" ts-insert-latency
+ load_and_generate_plot "${TS_INSERT_LATENCY}%" ts-insert-latency "latency (usec)"
- load_and_generate_plot "${TS_8THREAD_BENCH}%_latency" ts-8thread-insert-latency
- load_and_generate_plot "${TS_8THREAD_BENCH}%_throughput_%" ts-8thread-insert-throughput
+ load_and_generate_plot "${TS_8THREAD_BENCH}%_latency" ts-8thread-insert-latency "latency (usec)"
+ load_and_generate_plot "${TS_8THREAD_BENCH}_throughput" ts-8thread-insert-throughput "throughput (row/sec)"
+ load_and_generate_plot "${TS_8THREAD_BENCH}_cpu_efficiency" ts-8thread-insert-cpu-efficiency "CPU efficiency (row/cpu-sec)"
load_and_generate_plot "${FS_SCANINSERT_MRS}%_insert" fs-mrsonly-insert
load_and_generate_plot "${FS_SCANINSERT_MRS}%_scan%" fs-mrsonly-scan
@@ -668,18 +675,18 @@ load_stats_and_generate_plots() {
load_and_generate_plot "${DENSE_NODE_ITEST}_num_threads%" dense-node-bench-threads
load_and_generate_plot "${DENSE_NODE_ITEST}_num_bytes%" dense-node-bench-bytes
- load_and_generate_plot "${BLOCKING_QUEUE_SYMMETRIC_TEST}%" blocking-queue-symmetric
- load_and_generate_plot "${BLOCKING_QUEUE_NON_SYMMETRIC_TEST}%" blocking-queue-non-symmetric
+ load_and_generate_plot "${BLOCKING_QUEUE_SYMMETRIC_TEST}%" blocking-queue-symmetric "function call rate (req/sec)"
+ load_and_generate_plot "${BLOCKING_QUEUE_NON_SYMMETRIC_TEST}%" blocking-queue-non-symmetric "function call rate (req/sec)"
- load_and_generate_plot "${GET_TABLE_SCHEMA_RPC}%_req_rate" get-table-schema-rpc
- load_and_generate_plot "${GET_TABLE_SCHEMA_DIRECT_CALL}_authz_req_rate" get-table-schema-dc-authz
- load_and_generate_plot "${GET_TABLE_SCHEMA_DIRECT_CALL}_no_authz_req_rate" get-table-schema-dc-no-authz
+ load_and_generate_plot "${GET_TABLE_SCHEMA_RPC}%_req_rate" get-table-schema-rpc "RPC rate (req/sec)"
+ load_and_generate_plot "${GET_TABLE_SCHEMA_DIRECT_CALL}_authz_req_rate" get-table-schema-dc-authz "function call rate (req/sec)"
+ load_and_generate_plot "${GET_TABLE_SCHEMA_DIRECT_CALL}_no_authz_req_rate" get-table-schema-dc-no-authz "function call rate (req/sec)"
- load_and_generate_plot "${GET_TABLE_LOCATIONS_RPC}%_req_rate" get-table-locations-rpc
- load_and_generate_plot "${GET_TABLE_LOCATIONS_DIRECT_CALL}%_req_rate" get-table-locations-dc
+ load_and_generate_plot "${GET_TABLE_LOCATIONS_RPC}%_req_rate" get-table-locations-rpc "RPC rate (req/sec)"
+ load_and_generate_plot "${GET_TABLE_LOCATIONS_DIRECT_CALL}%_req_rate" get-table-locations-dc "function call rate (req/sec)"
- load_and_generate_plot "${SAME_TABLET_CONCURRENT_WRITES}_req_rate" same-tablet-concurrent-writes-rate
- load_and_generate_plot "${SAME_TABLET_CONCURRENT_WRITES}_overflows" same-tablet-concurrent-writes-overflows
+ load_and_generate_plot "${SAME_TABLET_CONCURRENT_WRITES}_req_rate" same-tablet-concurrent-writes-rate "RPC rate (req/sec)"
+ load_and_generate_plot "${SAME_TABLET_CONCURRENT_WRITES}_overflows" same-tablet-concurrent-writes-overflows "RPC queue overflows (total count)"
# Generate all the pngs for all the mt-tablet tests
for i in $(seq 0 $NUM_MT_TABLET_TESTS); do
[kudu] 02/03: [scripts] alternative y-label in jobs_runtime.R
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit f3c7b3f866d3f1f6505264e27b035e8d2798c2d1
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Tue Aug 4 19:06:44 2020 -0700
[scripts] alternative y-label in jobs_runtime.R
Added alternative y-label for graphs generated by jobs_runtime.R.
I'm planning to use this in a follow-up changelist.
Change-Id: I8f53c39348a61b8d54f22cfd9adaac20006f898a
Reviewed-on: http://gerrit.cloudera.org:8080/16289
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <aw...@cloudera.com>
---
src/kudu/scripts/jobs_runtime.R | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/src/kudu/scripts/jobs_runtime.R b/src/kudu/scripts/jobs_runtime.R
index e6dc97e..365d1d2 100644
--- a/src/kudu/scripts/jobs_runtime.R
+++ b/src/kudu/scripts/jobs_runtime.R
@@ -32,17 +32,25 @@ newpng <- function(filename = "img.png", width = 1500, height = 500) {
}
args <- commandArgs(trailingOnly = TRUE)
-if (length(args) < 2) {
- stop("usage: jobs_runtime.R <filename> <testname>")
+if (length(args) < 2 || length(args) > 3) {
+ stop("usage: jobs_runtime.R <filename> <testname> [y-axis-label]")
}
filename = args[1]
testname = args[2]
+ylabel="runtime"
+if (length(args) == 3) {
+ ylabel=args[3]
+}
newpng(paste(testname, "-jobs-runtime.png", sep = ""))
d <- read.table(file=filename, header=T)
print(ggplot(d, aes(x = build_number, y = runtime, color = workload)) +
+ ylab(ylabel) +
stat_summary(aes(group = workload), fun.y=median, geom = "line") +
- geom_boxplot(aes(group = interaction(workload, build_number)), position = "identity", outlier.size = 1.7, outlier.colour = "gray32") +
+ geom_boxplot(aes(group = interaction(workload, build_number)),
+ position = "identity",
+ outlier.size = 1.7,
+ outlier.colour = "gray32") +
ggtitle(testname))
[kudu] 01/03: [java] add state in ScanRequest toString
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 433c1771f05d25474554e1dd628f6a3d89c8f704
Author: Li Zhiming <li...@thinkingdata.cn>
AuthorDate: Tue Aug 4 12:00:58 2020 +0800
[java] add state in ScanRequest toString
Change-Id: Id5b90c8352eb36406663a0a60041462a57ffde76
Signed-off-by: Li Zhiming <li...@thinkingdata.cn>
Reviewed-on: http://gerrit.cloudera.org:8080/16281
Reviewed-by: Grant Henke <gr...@apache.org>
Tested-by: Alexey Serbin <as...@cloudera.com>
Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
.../src/main/java/org/apache/kudu/client/AsyncKuduScanner.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
index 3d1f0fa..c736bb5 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduScanner.java
@@ -1019,7 +1019,7 @@ public final class AsyncKuduScanner {
*/
final class ScanRequest extends KuduRpc<Response> {
- State state;
+ private final State state;
/** The token with which to authorize this RPC. */
private Token.SignedTokenPB authzToken;
@@ -1217,6 +1217,7 @@ public final class AsyncKuduScanner {
@Override
public String toString() {
return "ScanRequest(scannerId=" + Bytes.pretty(scannerId) +
+ ", state=" + state +
(tablet != null ? ", tablet=" + tablet.getTabletId() : "") +
", attempt=" + attempt + ", " + super.toString() + ")";
}