You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by li...@apache.org on 2022/05/06 10:51:34 UTC
[flink-ml] branch master updated: [FLINK-27096] Add a script to visualize benchmark results
This is an automated email from the ASF dual-hosted git repository.
lindong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-ml.git
The following commit(s) were added to refs/heads/master by this push:
new 2da5d59 [FLINK-27096] Add a script to visualize benchmark results
2da5d59 is described below
commit 2da5d59b2aa9e1c654ac129376bbf00fe9489615
Author: yunfengzhou-hub <yu...@outlook.com>
AuthorDate: Wed Apr 20 14:26:25 2022 +0800
[FLINK-27096] Add a script to visualize benchmark results
This PR also enriches benchmark result file's content with input parameters
This closes #87.
---
.../benchmark_results_visualization_example.svg | 1131 ++++++++++++++++++++
flink-ml-benchmark/README.md | 131 ++-
.../org/apache/flink/ml/benchmark/Benchmark.java | 88 +-
.../apache/flink/ml/benchmark/BenchmarkResult.java | 15 +
.../apache/flink/ml/benchmark/BenchmarkUtils.java | 50 +-
.../src/main/resources/benchmark-conf.json | 149 +++
.../apache/flink/ml/benchmark/BenchmarkTest.java | 4 +-
.../org/apache/flink/ml/util/ReadWriteUtils.java | 4 +-
.../bin/benchmark-results-visualize.py | 68 ++
.../{flink-ml-benchmark.sh => benchmark-run.sh} | 0
10 files changed, 1522 insertions(+), 118 deletions(-)
diff --git a/docs/static/fig/benchmark_results_visualization_example.svg b/docs/static/fig/benchmark_results_visualization_example.svg
new file mode 100644
index 0000000..a5093ad
--- /dev/null
+++ b/docs/static/fig/benchmark_results_visualization_example.svg
@@ -0,0 +1,1131 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Created with matplotlib (https://matplotlib.org/) -->
+<svg height="345.6pt" version="1.1" viewBox="0 0 460.8 345.6" width="460.8pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <defs>
+ <style type="text/css">
+*{stroke-linecap:butt;stroke-linejoin:round;}
+ </style>
+ </defs>
+ <g id="figure_1">
+ <g id="patch_1">
+ <path d="M 0 345.6
+L 460.8 345.6
+L 460.8 0
+L 0 0
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="axes_1">
+ <g id="patch_2">
+ <path d="M 57.6 307.584
+L 414.72 307.584
+L 414.72 41.472
+L 57.6 41.472
+z
+" style="fill:#ffffff;"/>
+ </g>
+ <g id="PathCollection_1">
+ <defs>
+ <path d="M 0 3
+C 0.795609 3 1.55874 2.683901 2.12132 2.12132
+C 2.683901 1.55874 3 0.795609 3 0
+C 3 -0.795609 2.683901 -1.55874 2.12132 -2.12132
+C 1.55874 -2.683901 0.795609 -3 0 -3
+C -0.795609 -3 -1.55874 -2.683901 -2.12132 -2.12132
+C -2.683901 -1.55874 -3 -0.795609 -3 0
+C -3 0.795609 -2.683901 1.55874 -2.12132 2.12132
+C -1.55874 2.683901 -0.795609 3 0 3
+z
+" id="ma5f96d9147" style="stroke:#1f77b4;"/>
+ </defs>
+ <g clip-path="url(#p70ef7d15d8)">
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="154.996388" xlink:href="#ma5f96d9147" y="268.118199"/>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="73.832776" xlink:href="#ma5f96d9147" y="295.487993"/>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="317.323612" xlink:href="#ma5f96d9147" y="142.761318"/>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="236.16" xlink:href="#ma5f96d9147" y="119.168897"/>
+ <use style="fill:#1f77b4;stroke:#1f77b4;" x="398.487224" xlink:href="#ma5f96d9147" y="53.568007"/>
+ </g>
+ </g>
+ <g id="matplotlib.axis_1">
+ <g id="xtick_1">
+ <g id="line2d_1">
+ <defs>
+ <path d="M 0 0
+L 0 3.5
+" id="m5a5ec87815" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="73.832776" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_1">
+ <!-- 10000 -->
+ <defs>
+ <path d="M 12.40625 8.296875
+L 28.515625 8.296875
+L 28.515625 63.921875
+L 10.984375 60.40625
+L 10.984375 69.390625
+L 28.421875 72.90625
+L 38.28125 72.90625
+L 38.28125 8.296875
+L 54.390625 8.296875
+L 54.390625 0
+L 12.40625 0
+z
+" id="DejaVuSans-49"/>
+ <path d="M 31.78125 66.40625
+Q 24.171875 66.40625 20.328125 58.90625
+Q 16.5 51.421875 16.5 36.375
+Q 16.5 21.390625 20.328125 13.890625
+Q 24.171875 6.390625 31.78125 6.390625
+Q 39.453125 6.390625 43.28125 13.890625
+Q 47.125 21.390625 47.125 36.375
+Q 47.125 51.421875 43.28125 58.90625
+Q 39.453125 66.40625 31.78125 66.40625
+z
+M 31.78125 74.21875
+Q 44.046875 74.21875 50.515625 64.515625
+Q 56.984375 54.828125 56.984375 36.375
+Q 56.984375 17.96875 50.515625 8.265625
+Q 44.046875 -1.421875 31.78125 -1.421875
+Q 19.53125 -1.421875 13.0625 8.265625
+Q 6.59375 17.96875 6.59375 36.375
+Q 6.59375 54.828125 13.0625 64.515625
+Q 19.53125 74.21875 31.78125 74.21875
+z
+" id="DejaVuSans-48"/>
+ </defs>
+ <g transform="translate(57.926526 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_2">
+ <g id="line2d_2">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="114.414582" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_2">
+ <!-- 15000 -->
+ <defs>
+ <path d="M 10.796875 72.90625
+L 49.515625 72.90625
+L 49.515625 64.59375
+L 19.828125 64.59375
+L 19.828125 46.734375
+Q 21.96875 47.46875 24.109375 47.828125
+Q 26.265625 48.1875 28.421875 48.1875
+Q 40.625 48.1875 47.75 41.5
+Q 54.890625 34.8125 54.890625 23.390625
+Q 54.890625 11.625 47.5625 5.09375
+Q 40.234375 -1.421875 26.90625 -1.421875
+Q 22.3125 -1.421875 17.546875 -0.640625
+Q 12.796875 0.140625 7.71875 1.703125
+L 7.71875 11.625
+Q 12.109375 9.234375 16.796875 8.0625
+Q 21.484375 6.890625 26.703125 6.890625
+Q 35.15625 6.890625 40.078125 11.328125
+Q 45.015625 15.765625 45.015625 23.390625
+Q 45.015625 31 40.078125 35.4375
+Q 35.15625 39.890625 26.703125 39.890625
+Q 22.75 39.890625 18.8125 39.015625
+Q 14.890625 38.140625 10.796875 36.28125
+z
+" id="DejaVuSans-53"/>
+ </defs>
+ <g transform="translate(98.508332 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-53"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_3">
+ <g id="line2d_3">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="154.996388" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_3">
+ <!-- 20000 -->
+ <defs>
+ <path d="M 19.1875 8.296875
+L 53.609375 8.296875
+L 53.609375 0
+L 7.328125 0
+L 7.328125 8.296875
+Q 12.9375 14.109375 22.625 23.890625
+Q 32.328125 33.6875 34.8125 36.53125
+Q 39.546875 41.84375 41.421875 45.53125
+Q 43.3125 49.21875 43.3125 52.78125
+Q 43.3125 58.59375 39.234375 62.25
+Q 35.15625 65.921875 28.609375 65.921875
+Q 23.96875 65.921875 18.8125 64.3125
+Q 13.671875 62.703125 7.8125 59.421875
+L 7.8125 69.390625
+Q 13.765625 71.78125 18.9375 73
+Q 24.125 74.21875 28.421875 74.21875
+Q 39.75 74.21875 46.484375 68.546875
+Q 53.21875 62.890625 53.21875 53.421875
+Q 53.21875 48.921875 51.53125 44.890625
+Q 49.859375 40.875 45.40625 35.40625
+Q 44.1875 33.984375 37.640625 27.21875
+Q 31.109375 20.453125 19.1875 8.296875
+z
+" id="DejaVuSans-50"/>
+ </defs>
+ <g transform="translate(139.090138 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-50"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_4">
+ <g id="line2d_4">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="195.578194" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_4">
+ <!-- 25000 -->
+ <g transform="translate(179.671944 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-50"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-53"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_5">
+ <g id="line2d_5">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="236.16" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_5">
+ <!-- 30000 -->
+ <defs>
+ <path d="M 40.578125 39.3125
+Q 47.65625 37.796875 51.625 33
+Q 55.609375 28.21875 55.609375 21.1875
+Q 55.609375 10.40625 48.1875 4.484375
+Q 40.765625 -1.421875 27.09375 -1.421875
+Q 22.515625 -1.421875 17.65625 -0.515625
+Q 12.796875 0.390625 7.625 2.203125
+L 7.625 11.71875
+Q 11.71875 9.328125 16.59375 8.109375
+Q 21.484375 6.890625 26.8125 6.890625
+Q 36.078125 6.890625 40.9375 10.546875
+Q 45.796875 14.203125 45.796875 21.1875
+Q 45.796875 27.640625 41.28125 31.265625
+Q 36.765625 34.90625 28.71875 34.90625
+L 20.21875 34.90625
+L 20.21875 43.015625
+L 29.109375 43.015625
+Q 36.375 43.015625 40.234375 45.921875
+Q 44.09375 48.828125 44.09375 54.296875
+Q 44.09375 59.90625 40.109375 62.90625
+Q 36.140625 65.921875 28.71875 65.921875
+Q 24.65625 65.921875 20.015625 65.03125
+Q 15.375 64.15625 9.8125 62.3125
+L 9.8125 71.09375
+Q 15.4375 72.65625 20.34375 73.4375
+Q 25.25 74.21875 29.59375 74.21875
+Q 40.828125 74.21875 47.359375 69.109375
+Q 53.90625 64.015625 53.90625 55.328125
+Q 53.90625 49.265625 50.4375 45.09375
+Q 46.96875 40.921875 40.578125 39.3125
+z
+" id="DejaVuSans-51"/>
+ </defs>
+ <g transform="translate(220.25375 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-51"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_6">
+ <g id="line2d_6">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="276.741806" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_6">
+ <!-- 35000 -->
+ <g transform="translate(260.835556 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-51"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-53"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_7">
+ <g id="line2d_7">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="317.323612" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_7">
+ <!-- 40000 -->
+ <defs>
+ <path d="M 37.796875 64.3125
+L 12.890625 25.390625
+L 37.796875 25.390625
+z
+M 35.203125 72.90625
+L 47.609375 72.90625
+L 47.609375 25.390625
+L 58.015625 25.390625
+L 58.015625 17.1875
+L 47.609375 17.1875
+L 47.609375 0
+L 37.796875 0
+L 37.796875 17.1875
+L 4.890625 17.1875
+L 4.890625 26.703125
+z
+" id="DejaVuSans-52"/>
+ </defs>
+ <g transform="translate(301.417362 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-52"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_8">
+ <g id="line2d_8">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="357.905418" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_8">
+ <!-- 45000 -->
+ <g transform="translate(341.999168 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-52"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-53"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="xtick_9">
+ <g id="line2d_9">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="398.487224" xlink:href="#m5a5ec87815" y="307.584"/>
+ </g>
+ </g>
+ <g id="text_9">
+ <!-- 50000 -->
+ <g transform="translate(382.580974 322.182437)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-53"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_10">
+ <!-- inputData.paramMap.numValues -->
+ <defs>
+ <path d="M 9.421875 54.6875
+L 18.40625 54.6875
+L 18.40625 0
+L 9.421875 0
+z
+M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 64.59375
+L 9.421875 64.59375
+z
+" id="DejaVuSans-105"/>
+ <path d="M 54.890625 33.015625
+L 54.890625 0
+L 45.90625 0
+L 45.90625 32.71875
+Q 45.90625 40.484375 42.875 44.328125
+Q 39.84375 48.1875 33.796875 48.1875
+Q 26.515625 48.1875 22.3125 43.546875
+Q 18.109375 38.921875 18.109375 30.90625
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 21.34375 51.125 25.703125 53.5625
+Q 30.078125 56 35.796875 56
+Q 45.21875 56 50.046875 50.171875
+Q 54.890625 44.34375 54.890625 33.015625
+z
+" id="DejaVuSans-110"/>
+ <path d="M 18.109375 8.203125
+L 18.109375 -20.796875
+L 9.078125 -20.796875
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.390625
+Q 20.953125 51.265625 25.265625 53.625
+Q 29.59375 56 35.59375 56
+Q 45.5625 56 51.78125 48.09375
+Q 58.015625 40.1875 58.015625 27.296875
+Q 58.015625 14.40625 51.78125 6.484375
+Q 45.5625 -1.421875 35.59375 -1.421875
+Q 29.59375 -1.421875 25.265625 0.953125
+Q 20.953125 3.328125 18.109375 8.203125
+z
+M 48.6875 27.296875
+Q 48.6875 37.203125 44.609375 42.84375
+Q 40.53125 48.484375 33.40625 48.484375
+Q 26.265625 48.484375 22.1875 42.84375
+Q 18.109375 37.203125 18.109375 27.296875
+Q 18.109375 17.390625 22.1875 11.75
+Q 26.265625 6.109375 33.40625 6.109375
+Q 40.53125 6.109375 44.609375 11.75
+Q 48.6875 17.390625 48.6875 27.296875
+z
+" id="DejaVuSans-112"/>
+ <path d="M 8.5 21.578125
+L 8.5 54.6875
+L 17.484375 54.6875
+L 17.484375 21.921875
+Q 17.484375 14.15625 20.5 10.265625
+Q 23.53125 6.390625 29.59375 6.390625
+Q 36.859375 6.390625 41.078125 11.03125
+Q 45.3125 15.671875 45.3125 23.6875
+L 45.3125 54.6875
+L 54.296875 54.6875
+L 54.296875 0
+L 45.3125 0
+L 45.3125 8.40625
+Q 42.046875 3.421875 37.71875 1
+Q 33.40625 -1.421875 27.6875 -1.421875
+Q 18.265625 -1.421875 13.375 4.4375
+Q 8.5 10.296875 8.5 21.578125
+z
+M 31.109375 56
+z
+" id="DejaVuSans-117"/>
+ <path d="M 18.3125 70.21875
+L 18.3125 54.6875
+L 36.8125 54.6875
+L 36.8125 47.703125
+L 18.3125 47.703125
+L 18.3125 18.015625
+Q 18.3125 11.328125 20.140625 9.421875
+Q 21.96875 7.515625 27.59375 7.515625
+L 36.8125 7.515625
+L 36.8125 0
+L 27.59375 0
+Q 17.1875 0 13.234375 3.875
+Q 9.28125 7.765625 9.28125 18.015625
+L 9.28125 47.703125
+L 2.6875 47.703125
+L 2.6875 54.6875
+L 9.28125 54.6875
+L 9.28125 70.21875
+z
+" id="DejaVuSans-116"/>
+ <path d="M 19.671875 64.796875
+L 19.671875 8.109375
+L 31.59375 8.109375
+Q 46.6875 8.109375 53.6875 14.9375
+Q 60.6875 21.78125 60.6875 36.53125
+Q 60.6875 51.171875 53.6875 57.984375
+Q 46.6875 64.796875 31.59375 64.796875
+z
+M 9.8125 72.90625
+L 30.078125 72.90625
+Q 51.265625 72.90625 61.171875 64.09375
+Q 71.09375 55.28125 71.09375 36.53125
+Q 71.09375 17.671875 61.125 8.828125
+Q 51.171875 0 30.078125 0
+L 9.8125 0
+z
+" id="DejaVuSans-68"/>
+ <path d="M 34.28125 27.484375
+Q 23.390625 27.484375 19.1875 25
+Q 14.984375 22.515625 14.984375 16.5
+Q 14.984375 11.71875 18.140625 8.90625
+Q 21.296875 6.109375 26.703125 6.109375
+Q 34.1875 6.109375 38.703125 11.40625
+Q 43.21875 16.703125 43.21875 25.484375
+L 43.21875 27.484375
+z
+M 52.203125 31.203125
+L 52.203125 0
+L 43.21875 0
+L 43.21875 8.296875
+Q 40.140625 3.328125 35.546875 0.953125
+Q 30.953125 -1.421875 24.3125 -1.421875
+Q 15.921875 -1.421875 10.953125 3.296875
+Q 6 8.015625 6 15.921875
+Q 6 25.140625 12.171875 29.828125
+Q 18.359375 34.515625 30.609375 34.515625
+L 43.21875 34.515625
+L 43.21875 35.40625
+Q 43.21875 41.609375 39.140625 45
+Q 35.0625 48.390625 27.6875 48.390625
+Q 23 48.390625 18.546875 47.265625
+Q 14.109375 46.140625 10.015625 43.890625
+L 10.015625 52.203125
+Q 14.9375 54.109375 19.578125 55.046875
+Q 24.21875 56 28.609375 56
+Q 40.484375 56 46.34375 49.84375
+Q 52.203125 43.703125 52.203125 31.203125
+z
+" id="DejaVuSans-97"/>
+ <path d="M 10.6875 12.40625
+L 21 12.40625
+L 21 0
+L 10.6875 0
+z
+" id="DejaVuSans-46"/>
+ <path d="M 41.109375 46.296875
+Q 39.59375 47.171875 37.8125 47.578125
+Q 36.03125 48 33.890625 48
+Q 26.265625 48 22.1875 43.046875
+Q 18.109375 38.09375 18.109375 28.8125
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 20.953125 51.171875 25.484375 53.578125
+Q 30.03125 56 36.53125 56
+Q 37.453125 56 38.578125 55.875
+Q 39.703125 55.765625 41.0625 55.515625
+z
+" id="DejaVuSans-114"/>
+ <path d="M 52 44.1875
+Q 55.375 50.25 60.0625 53.125
+Q 64.75 56 71.09375 56
+Q 79.640625 56 84.28125 50.015625
+Q 88.921875 44.046875 88.921875 33.015625
+L 88.921875 0
+L 79.890625 0
+L 79.890625 32.71875
+Q 79.890625 40.578125 77.09375 44.375
+Q 74.3125 48.1875 68.609375 48.1875
+Q 61.625 48.1875 57.5625 43.546875
+Q 53.515625 38.921875 53.515625 30.90625
+L 53.515625 0
+L 44.484375 0
+L 44.484375 32.71875
+Q 44.484375 40.625 41.703125 44.40625
+Q 38.921875 48.1875 33.109375 48.1875
+Q 26.21875 48.1875 22.15625 43.53125
+Q 18.109375 38.875 18.109375 30.90625
+L 18.109375 0
+L 9.078125 0
+L 9.078125 54.6875
+L 18.109375 54.6875
+L 18.109375 46.1875
+Q 21.1875 51.21875 25.484375 53.609375
+Q 29.78125 56 35.6875 56
+Q 41.65625 56 45.828125 52.96875
+Q 50 49.953125 52 44.1875
+z
+" id="DejaVuSans-109"/>
+ <path d="M 9.8125 72.90625
+L 24.515625 72.90625
+L 43.109375 23.296875
+L 61.8125 72.90625
+L 76.515625 72.90625
+L 76.515625 0
+L 66.890625 0
+L 66.890625 64.015625
+L 48.09375 14.015625
+L 38.1875 14.015625
+L 19.390625 64.015625
+L 19.390625 0
+L 9.8125 0
+z
+" id="DejaVuSans-77"/>
+ <path d="M 28.609375 0
+L 0.78125 72.90625
+L 11.078125 72.90625
+L 34.1875 11.53125
+L 57.328125 72.90625
+L 67.578125 72.90625
+L 39.796875 0
+z
+" id="DejaVuSans-86"/>
+ <path d="M 9.421875 75.984375
+L 18.40625 75.984375
+L 18.40625 0
+L 9.421875 0
+z
+" id="DejaVuSans-108"/>
+ <path d="M 56.203125 29.59375
+L 56.203125 25.203125
+L 14.890625 25.203125
+Q 15.484375 15.921875 20.484375 11.0625
+Q 25.484375 6.203125 34.421875 6.203125
+Q 39.59375 6.203125 44.453125 7.46875
+Q 49.3125 8.734375 54.109375 11.28125
+L 54.109375 2.78125
+Q 49.265625 0.734375 44.1875 -0.34375
+Q 39.109375 -1.421875 33.890625 -1.421875
+Q 20.796875 -1.421875 13.15625 6.1875
+Q 5.515625 13.8125 5.515625 26.8125
+Q 5.515625 40.234375 12.765625 48.109375
+Q 20.015625 56 32.328125 56
+Q 43.359375 56 49.78125 48.890625
+Q 56.203125 41.796875 56.203125 29.59375
+z
+M 47.21875 32.234375
+Q 47.125 39.59375 43.09375 43.984375
+Q 39.0625 48.390625 32.421875 48.390625
+Q 24.90625 48.390625 20.390625 44.140625
+Q 15.875 39.890625 15.1875 32.171875
+z
+" id="DejaVuSans-101"/>
+ <path d="M 44.28125 53.078125
+L 44.28125 44.578125
+Q 40.484375 46.53125 36.375 47.5
+Q 32.28125 48.484375 27.875 48.484375
+Q 21.1875 48.484375 17.84375 46.4375
+Q 14.5 44.390625 14.5 40.28125
+Q 14.5 37.15625 16.890625 35.375
+Q 19.28125 33.59375 26.515625 31.984375
+L 29.59375 31.296875
+Q 39.15625 29.25 43.1875 25.515625
+Q 47.21875 21.78125 47.21875 15.09375
+Q 47.21875 7.46875 41.1875 3.015625
+Q 35.15625 -1.421875 24.609375 -1.421875
+Q 20.21875 -1.421875 15.453125 -0.5625
+Q 10.6875 0.296875 5.421875 2
+L 5.421875 11.28125
+Q 10.40625 8.6875 15.234375 7.390625
+Q 20.0625 6.109375 24.8125 6.109375
+Q 31.15625 6.109375 34.5625 8.28125
+Q 37.984375 10.453125 37.984375 14.40625
+Q 37.984375 18.0625 35.515625 20.015625
+Q 33.0625 21.96875 24.703125 23.78125
+L 21.578125 24.515625
+Q 13.234375 26.265625 9.515625 29.90625
+Q 5.8125 33.546875 5.8125 39.890625
+Q 5.8125 47.609375 11.28125 51.796875
+Q 16.75 56 26.8125 56
+Q 31.78125 56 36.171875 55.265625
+Q 40.578125 54.546875 44.28125 53.078125
+z
+" id="DejaVuSans-115"/>
+ </defs>
+ <g transform="translate(153.476406 335.860562)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-105"/>
+ <use x="27.783203" xlink:href="#DejaVuSans-110"/>
+ <use x="91.162109" xlink:href="#DejaVuSans-112"/>
+ <use x="154.638672" xlink:href="#DejaVuSans-117"/>
+ <use x="218.017578" xlink:href="#DejaVuSans-116"/>
+ <use x="257.226562" xlink:href="#DejaVuSans-68"/>
+ <use x="334.228516" xlink:href="#DejaVuSans-97"/>
+ <use x="395.507812" xlink:href="#DejaVuSans-116"/>
+ <use x="434.716797" xlink:href="#DejaVuSans-97"/>
+ <use x="495.996094" xlink:href="#DejaVuSans-46"/>
+ <use x="527.783203" xlink:href="#DejaVuSans-112"/>
+ <use x="591.259766" xlink:href="#DejaVuSans-97"/>
+ <use x="652.539062" xlink:href="#DejaVuSans-114"/>
+ <use x="693.652344" xlink:href="#DejaVuSans-97"/>
+ <use x="754.931641" xlink:href="#DejaVuSans-109"/>
+ <use x="852.34375" xlink:href="#DejaVuSans-77"/>
+ <use x="938.623047" xlink:href="#DejaVuSans-97"/>
+ <use x="999.902344" xlink:href="#DejaVuSans-112"/>
+ <use x="1063.378906" xlink:href="#DejaVuSans-46"/>
+ <use x="1095.166016" xlink:href="#DejaVuSans-110"/>
+ <use x="1158.544922" xlink:href="#DejaVuSans-117"/>
+ <use x="1221.923828" xlink:href="#DejaVuSans-109"/>
+ <use x="1319.335938" xlink:href="#DejaVuSans-86"/>
+ <use x="1387.634766" xlink:href="#DejaVuSans-97"/>
+ <use x="1448.914062" xlink:href="#DejaVuSans-108"/>
+ <use x="1476.697266" xlink:href="#DejaVuSans-117"/>
+ <use x="1540.076172" xlink:href="#DejaVuSans-101"/>
+ <use x="1601.599609" xlink:href="#DejaVuSans-115"/>
+ </g>
+ </g>
+ </g>
+ <g id="matplotlib.axis_2">
+ <g id="ytick_1">
+ <g id="line2d_10">
+ <defs>
+ <path d="M 0 0
+L -3.5 0
+" id="m2e4e04fa87" style="stroke:#000000;stroke-width:0.8;"/>
+ </defs>
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m2e4e04fa87" y="300.862426"/>
+ </g>
+ </g>
+ <g id="text_11">
+ <!-- 100000 -->
+ <g transform="translate(12.425 304.661644)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ <use x="318.115234" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_2">
+ <g id="line2d_11">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m2e4e04fa87" y="258.762705"/>
+ </g>
+ </g>
+ <g id="text_12">
+ <!-- 150000 -->
+ <g transform="translate(12.425 262.561924)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-49"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-53"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ <use x="318.115234" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_3">
+ <g id="line2d_12">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m2e4e04fa87" y="216.662985"/>
+ </g>
+ </g>
+ <g id="text_13">
+ <!-- 200000 -->
+ <g transform="translate(12.425 220.462204)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-50"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ <use x="318.115234" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_4">
+ <g id="line2d_13">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m2e4e04fa87" y="174.563265"/>
+ </g>
+ </g>
+ <g id="text_14">
+ <!-- 250000 -->
+ <g transform="translate(12.425 178.362484)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-50"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-53"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ <use x="318.115234" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_5">
+ <g id="line2d_14">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m2e4e04fa87" y="132.463545"/>
+ </g>
+ </g>
+ <g id="text_15">
+ <!-- 300000 -->
+ <g transform="translate(12.425 136.262764)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-51"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ <use x="318.115234" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_6">
+ <g id="line2d_15">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m2e4e04fa87" y="90.363825"/>
+ </g>
+ </g>
+ <g id="text_16">
+ <!-- 350000 -->
+ <g transform="translate(12.425 94.163044)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-51"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-53"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ <use x="318.115234" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="ytick_7">
+ <g id="line2d_16">
+ <g>
+ <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m2e4e04fa87" y="48.264105"/>
+ </g>
+ </g>
+ <g id="text_17">
+ <!-- 400000 -->
+ <g transform="translate(12.425 52.063324)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-52"/>
+ <use x="63.623047" xlink:href="#DejaVuSans-48"/>
+ <use x="127.246094" xlink:href="#DejaVuSans-48"/>
+ <use x="190.869141" xlink:href="#DejaVuSans-48"/>
+ <use x="254.492188" xlink:href="#DejaVuSans-48"/>
+ <use x="318.115234" xlink:href="#DejaVuSans-48"/>
+ </g>
+ </g>
+ </g>
+ <g id="text_18">
+ <!-- results.inputThroughput -->
+ <defs>
+ <path d="M -0.296875 72.90625
+L 61.375 72.90625
+L 61.375 64.59375
+L 35.5 64.59375
+L 35.5 0
+L 25.59375 0
+L 25.59375 64.59375
+L -0.296875 64.59375
+z
+" id="DejaVuSans-84"/>
+ <path d="M 54.890625 33.015625
+L 54.890625 0
+L 45.90625 0
+L 45.90625 32.71875
+Q 45.90625 40.484375 42.875 44.328125
+Q 39.84375 48.1875 33.796875 48.1875
+Q 26.515625 48.1875 22.3125 43.546875
+Q 18.109375 38.921875 18.109375 30.90625
+L 18.109375 0
+L 9.078125 0
+L 9.078125 75.984375
+L 18.109375 75.984375
+L 18.109375 46.1875
+Q 21.34375 51.125 25.703125 53.5625
+Q 30.078125 56 35.796875 56
+Q 45.21875 56 50.046875 50.171875
+Q 54.890625 44.34375 54.890625 33.015625
+z
+" id="DejaVuSans-104"/>
+ <path d="M 30.609375 48.390625
+Q 23.390625 48.390625 19.1875 42.75
+Q 14.984375 37.109375 14.984375 27.296875
+Q 14.984375 17.484375 19.15625 11.84375
+Q 23.34375 6.203125 30.609375 6.203125
+Q 37.796875 6.203125 41.984375 11.859375
+Q 46.1875 17.53125 46.1875 27.296875
+Q 46.1875 37.015625 41.984375 42.703125
+Q 37.796875 48.390625 30.609375 48.390625
+z
+M 30.609375 56
+Q 42.328125 56 49.015625 48.375
+Q 55.71875 40.765625 55.71875 27.296875
+Q 55.71875 13.875 49.015625 6.21875
+Q 42.328125 -1.421875 30.609375 -1.421875
+Q 18.84375 -1.421875 12.171875 6.21875
+Q 5.515625 13.875 5.515625 27.296875
+Q 5.515625 40.765625 12.171875 48.375
+Q 18.84375 56 30.609375 56
+z
+" id="DejaVuSans-111"/>
+ <path d="M 45.40625 27.984375
+Q 45.40625 37.75 41.375 43.109375
+Q 37.359375 48.484375 30.078125 48.484375
+Q 22.859375 48.484375 18.828125 43.109375
+Q 14.796875 37.75 14.796875 27.984375
+Q 14.796875 18.265625 18.828125 12.890625
+Q 22.859375 7.515625 30.078125 7.515625
+Q 37.359375 7.515625 41.375 12.890625
+Q 45.40625 18.265625 45.40625 27.984375
+z
+M 54.390625 6.78125
+Q 54.390625 -7.171875 48.1875 -13.984375
+Q 42 -20.796875 29.203125 -20.796875
+Q 24.46875 -20.796875 20.265625 -20.09375
+Q 16.0625 -19.390625 12.109375 -17.921875
+L 12.109375 -9.1875
+Q 16.0625 -11.328125 19.921875 -12.34375
+Q 23.78125 -13.375 27.78125 -13.375
+Q 36.625 -13.375 41.015625 -8.765625
+Q 45.40625 -4.15625 45.40625 5.171875
+L 45.40625 9.625
+Q 42.625 4.78125 38.28125 2.390625
+Q 33.9375 0 27.875 0
+Q 17.828125 0 11.671875 7.65625
+Q 5.515625 15.328125 5.515625 27.984375
+Q 5.515625 40.671875 11.671875 48.328125
+Q 17.828125 56 27.875 56
+Q 33.9375 56 38.28125 53.609375
+Q 42.625 51.21875 45.40625 46.390625
+L 45.40625 54.6875
+L 54.390625 54.6875
+z
+" id="DejaVuSans-103"/>
+ </defs>
+ <g transform="translate(6.345312 234.986594)rotate(-90)scale(0.1 -0.1)">
+ <use xlink:href="#DejaVuSans-114"/>
+ <use x="41.082031" xlink:href="#DejaVuSans-101"/>
+ <use x="102.605469" xlink:href="#DejaVuSans-115"/>
+ <use x="154.705078" xlink:href="#DejaVuSans-117"/>
+ <use x="218.083984" xlink:href="#DejaVuSans-108"/>
+ <use x="245.867188" xlink:href="#DejaVuSans-116"/>
+ <use x="285.076172" xlink:href="#DejaVuSans-115"/>
+ <use x="337.175781" xlink:href="#DejaVuSans-46"/>
+ <use x="368.962891" xlink:href="#DejaVuSans-105"/>
+ <use x="396.746094" xlink:href="#DejaVuSans-110"/>
+ <use x="460.125" xlink:href="#DejaVuSans-112"/>
+ <use x="523.601562" xlink:href="#DejaVuSans-117"/>
+ <use x="586.980469" xlink:href="#DejaVuSans-116"/>
+ <use x="626.189453" xlink:href="#DejaVuSans-84"/>
+ <use x="687.273438" xlink:href="#DejaVuSans-104"/>
+ <use x="750.652344" xlink:href="#DejaVuSans-114"/>
+ <use x="791.734375" xlink:href="#DejaVuSans-111"/>
+ <use x="852.916016" xlink:href="#DejaVuSans-117"/>
+ <use x="916.294922" xlink:href="#DejaVuSans-103"/>
+ <use x="979.771484" xlink:href="#DejaVuSans-104"/>
+ <use x="1043.150391" xlink:href="#DejaVuSans-112"/>
+ <use x="1106.626953" xlink:href="#DejaVuSans-117"/>
+ <use x="1170.005859" xlink:href="#DejaVuSans-116"/>
+ </g>
+ </g>
+ </g>
+ <g id="patch_3">
+ <path d="M 57.6 307.584
+L 57.6 41.472
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_4">
+ <path d="M 414.72 307.584
+L 414.72 41.472
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_5">
+ <path d="M 57.6 307.584
+L 414.72 307.584
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="patch_6">
+ <path d="M 57.6 41.472
+L 414.72 41.472
+" style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
+ </g>
+ <g id="text_19">
+ <!-- Flink ML Benchmark Results Visualization -->
+ <defs>
+ <path d="M 9.8125 72.90625
+L 51.703125 72.90625
+L 51.703125 64.59375
+L 19.671875 64.59375
+L 19.671875 43.109375
+L 48.578125 43.109375
+L 48.578125 34.8125
+L 19.671875 34.8125
+L 19.671875 0
+L 9.8125 0
+z
+" id="DejaVuSans-70"/>
+ <path d="M 9.078125 75.984375
+L 18.109375 75.984375
+L 18.109375 31.109375
+L 44.921875 54.6875
+L 56.390625 54.6875
+L 27.390625 29.109375
+L 57.625 0
+L 45.90625 0
+L 18.109375 26.703125
+L 18.109375 0
+L 9.078125 0
+z
+" id="DejaVuSans-107"/>
+ <path id="DejaVuSans-32"/>
+ <path d="M 9.8125 72.90625
+L 19.671875 72.90625
+L 19.671875 8.296875
+L 55.171875 8.296875
+L 55.171875 0
+L 9.8125 0
+z
+" id="DejaVuSans-76"/>
+ <path d="M 19.671875 34.8125
+L 19.671875 8.109375
+L 35.5 8.109375
+Q 43.453125 8.109375 47.28125 11.40625
+Q 51.125 14.703125 51.125 21.484375
+Q 51.125 28.328125 47.28125 31.5625
+Q 43.453125 34.8125 35.5 34.8125
+z
+M 19.671875 64.796875
+L 19.671875 42.828125
+L 34.28125 42.828125
+Q 41.5 42.828125 45.03125 45.53125
+Q 48.578125 48.25 48.578125 53.8125
+Q 48.578125 59.328125 45.03125 62.0625
+Q 41.5 64.796875 34.28125 64.796875
+z
+M 9.8125 72.90625
+L 35.015625 72.90625
+Q 46.296875 72.90625 52.390625 68.21875
+Q 58.5 63.53125 58.5 54.890625
+Q 58.5 48.1875 55.375 44.234375
+Q 52.25 40.28125 46.1875 39.3125
+Q 53.46875 37.75 57.5 32.78125
+Q 61.53125 27.828125 61.53125 20.40625
+Q 61.53125 10.640625 54.890625 5.3125
+Q 48.25 0 35.984375 0
+L 9.8125 0
+z
+" id="DejaVuSans-66"/>
+ <path d="M 48.78125 52.59375
+L 48.78125 44.1875
+Q 44.96875 46.296875 41.140625 47.34375
+Q 37.3125 48.390625 33.40625 48.390625
+Q 24.65625 48.390625 19.8125 42.84375
+Q 14.984375 37.3125 14.984375 27.296875
+Q 14.984375 17.28125 19.8125 11.734375
+Q 24.65625 6.203125 33.40625 6.203125
+Q 37.3125 6.203125 41.140625 7.25
+Q 44.96875 8.296875 48.78125 10.40625
+L 48.78125 2.09375
+Q 45.015625 0.34375 40.984375 -0.53125
+Q 36.96875 -1.421875 32.421875 -1.421875
+Q 20.0625 -1.421875 12.78125 6.34375
+Q 5.515625 14.109375 5.515625 27.296875
+Q 5.515625 40.671875 12.859375 48.328125
+Q 20.21875 56 33.015625 56
+Q 37.15625 56 41.109375 55.140625
+Q 45.0625 54.296875 48.78125 52.59375
+z
+" id="DejaVuSans-99"/>
+ <path d="M 44.390625 34.1875
+Q 47.5625 33.109375 50.5625 29.59375
+Q 53.5625 26.078125 56.59375 19.921875
+L 66.609375 0
+L 56 0
+L 46.6875 18.703125
+Q 43.0625 26.03125 39.671875 28.421875
+Q 36.28125 30.8125 30.421875 30.8125
+L 19.671875 30.8125
+L 19.671875 0
+L 9.8125 0
+L 9.8125 72.90625
+L 32.078125 72.90625
+Q 44.578125 72.90625 50.734375 67.671875
+Q 56.890625 62.453125 56.890625 51.90625
+Q 56.890625 45.015625 53.6875 40.46875
+Q 50.484375 35.9375 44.390625 34.1875
+z
+M 19.671875 64.796875
+L 19.671875 38.921875
+L 32.078125 38.921875
+Q 39.203125 38.921875 42.84375 42.21875
+Q 46.484375 45.515625 46.484375 51.90625
+Q 46.484375 58.296875 42.84375 61.546875
+Q 39.203125 64.796875 32.078125 64.796875
+z
+" id="DejaVuSans-82"/>
+ <path d="M 5.515625 54.6875
+L 48.1875 54.6875
+L 48.1875 46.484375
+L 14.40625 7.171875
+L 48.1875 7.171875
+L 48.1875 0
+L 4.296875 0
+L 4.296875 8.203125
+L 38.09375 47.515625
+L 5.515625 47.515625
+z
+" id="DejaVuSans-122"/>
+ </defs>
+ <g transform="translate(111.819375 35.472)scale(0.12 -0.12)">
+ <use xlink:href="#DejaVuSans-70"/>
+ <use x="57.519531" xlink:href="#DejaVuSans-108"/>
+ <use x="85.302734" xlink:href="#DejaVuSans-105"/>
+ <use x="113.085938" xlink:href="#DejaVuSans-110"/>
+ <use x="176.464844" xlink:href="#DejaVuSans-107"/>
+ <use x="234.375" xlink:href="#DejaVuSans-32"/>
+ <use x="266.162109" xlink:href="#DejaVuSans-77"/>
+ <use x="352.441406" xlink:href="#DejaVuSans-76"/>
+ <use x="408.154297" xlink:href="#DejaVuSans-32"/>
+ <use x="439.941406" xlink:href="#DejaVuSans-66"/>
+ <use x="508.544922" xlink:href="#DejaVuSans-101"/>
+ <use x="570.068359" xlink:href="#DejaVuSans-110"/>
+ <use x="633.447266" xlink:href="#DejaVuSans-99"/>
+ <use x="688.427734" xlink:href="#DejaVuSans-104"/>
+ <use x="751.806641" xlink:href="#DejaVuSans-109"/>
+ <use x="849.21875" xlink:href="#DejaVuSans-97"/>
+ <use x="910.498047" xlink:href="#DejaVuSans-114"/>
+ <use x="951.611328" xlink:href="#DejaVuSans-107"/>
+ <use x="1009.521484" xlink:href="#DejaVuSans-32"/>
+ <use x="1041.308594" xlink:href="#DejaVuSans-82"/>
+ <use x="1110.728516" xlink:href="#DejaVuSans-101"/>
+ <use x="1172.251953" xlink:href="#DejaVuSans-115"/>
+ <use x="1224.351562" xlink:href="#DejaVuSans-117"/>
+ <use x="1287.730469" xlink:href="#DejaVuSans-108"/>
+ <use x="1315.513672" xlink:href="#DejaVuSans-116"/>
+ <use x="1354.722656" xlink:href="#DejaVuSans-115"/>
+ <use x="1406.822266" xlink:href="#DejaVuSans-32"/>
+ <use x="1438.609375" xlink:href="#DejaVuSans-86"/>
+ <use x="1506.986328" xlink:href="#DejaVuSans-105"/>
+ <use x="1534.769531" xlink:href="#DejaVuSans-115"/>
+ <use x="1586.869141" xlink:href="#DejaVuSans-117"/>
+ <use x="1650.248047" xlink:href="#DejaVuSans-97"/>
+ <use x="1711.527344" xlink:href="#DejaVuSans-108"/>
+ <use x="1739.310547" xlink:href="#DejaVuSans-105"/>
+ <use x="1767.09375" xlink:href="#DejaVuSans-122"/>
+ <use x="1819.583984" xlink:href="#DejaVuSans-97"/>
+ <use x="1880.863281" xlink:href="#DejaVuSans-116"/>
+ <use x="1920.072266" xlink:href="#DejaVuSans-105"/>
+ <use x="1947.855469" xlink:href="#DejaVuSans-111"/>
+ <use x="2009.037109" xlink:href="#DejaVuSans-110"/>
+ </g>
+ </g>
+ </g>
+ </g>
+ <defs>
+ <clipPath id="p70ef7d15d8">
+ <rect height="266.112" width="357.12" x="57.6" y="41.472"/>
+ </clipPath>
+ </defs>
+</svg>
diff --git a/flink-ml-benchmark/README.md b/flink-ml-benchmark/README.md
index ac57722..3966f4c 100644
--- a/flink-ml-benchmark/README.md
+++ b/flink-ml-benchmark/README.md
@@ -48,7 +48,7 @@ cp ./lib/*.jar $FLINK_HOME/lib/
## Run Benchmark Example
-Please start a Flink standalone session in your local environment with the
+Please start a Flink standalone cluster in your local environment with the
following command.
```bash
@@ -63,7 +63,7 @@ Then in Flink ML's binary distribution's folder, execute the following command
to run the default benchmarks.
```bash
-./bin/flink-ml-benchmark.sh ./conf/benchmark-conf.json --output-file results.json
+./bin/benchmark-run.sh conf/benchmark-conf.json --output-file results.json
```
You will notice that some Flink jobs are submitted to your Flink cluster, and
@@ -71,52 +71,82 @@ the following information is printed out in your terminal. This means that you
have successfully executed the default benchmarks.
```
-Found benchmarks [KMeansModel-1, KMeans-1]
+Found 8 benchmarks.
+
...
+
Benchmarks execution completed.
-Benchmark results summary:
-[ {
- "name" : "KMeansModel-1",
- "totalTimeMs" : 782.0,
- "inputRecordNum" : 10000,
- "inputThroughput" : 12787.723785166241,
- "outputRecordNum" : 10000,
- "outputThroughput" : 12787.723785166241
-}, {
- "name" : "KMeans-1",
- "totalTimeMs" : 7022.0,
- "inputRecordNum" : 10000,
- "inputThroughput" : 1424.0956992309884,
- "outputRecordNum" : 1,
- "outputThroughput" : 0.14240956992309883
-} ]
Benchmark results saved as json in results.json.
```
-The command above would save the results into `results.json` as below.
+The command above would save the results into `results.json` as a map. For a
+successfully executed benchmark, a map entry of the following format would be
+added.
```json
-[ {
- "name" : "KMeansModel-1",
- "totalTimeMs" : 782.0,
- "inputRecordNum" : 10000,
- "inputThroughput" : 12787.723785166241,
- "outputRecordNum" : 10000,
- "outputThroughput" : 12787.723785166241
-}, {
- "name" : "KMeans-1",
- "totalTimeMs" : 7022.0,
- "inputRecordNum" : 10000,
- "inputThroughput" : 1424.0956992309884,
- "outputRecordNum" : 1,
- "outputThroughput" : 0.14240956992309883
-} ]
+{
+ "KMeans-1" : {
+ "stage" : {
+ "className" : "org.apache.flink.ml.clustering.kmeans.KMeans",
+ "paramMap" : {
+ "featuresCol" : "features",
+ "predictionCol" : "prediction"
+ }
+ },
+ "inputData" : {
+ "className" : "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap" : {
+ "seed" : 2,
+ "colNames" : [ "features" ],
+ "numValues" : 10000,
+ "vectorDim" : 10
+ }
+ },
+ "results" : {
+ "totalTimeMs" : 7148.0,
+ "inputRecordNum" : 10000,
+ "inputThroughput" : 1398.9927252378288,
+ "outputRecordNum" : 1,
+ "outputThroughput" : 0.13989927252378287
+ }
+ }
+}
```
-## Customize Benchmark Configuration
+If a benchmark failed, it would be saved in the file as follows.
-`flink-ml-benchmark.sh` parses benchmarks to be executed according to the input
-configuration file, like `./conf/benchmark-conf.json`. It can also parse your
+```json
+{
+ "Unmatch-Input" : {
+ "stage" : {
+ "className" : "org.apache.flink.ml.clustering.kmeans.KMeans",
+ "paramMap" : {
+ "featuresCol" : "features",
+ "predictionCol" : "prediction"
+ }
+ },
+ "inputData" : {
+ "className" : "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap" : {
+ "seed" : 2,
+ "colNames" : [ "non-features" ],
+ "numValues" : 10000,
+ "vectorDim" : 10
+ }
+ },
+ "results" : {
+ "exception" : "java.lang.NullPointerException(ReadWriteUtils.java:388)"
+ }
+ }
+}
+```
+
+## Advanced Topics
+
+### Customize Benchmark Configuration
+
+`benchmark-run.sh` parses benchmarks to be executed according to the input
+configuration file, like `conf/benchmark-conf.json`. It can also parse your
custom configuration file so long as it contains a JSON object in the following
format.
@@ -138,10 +168,10 @@ format.
- `"paramMap"`: A JSON object containing the parameters related to the
specific `Stage` or `DataGenerator`.
-Combining the format requirements above, the example configuration in
-`./conf/benchmark-conf.json` is as follows. This configuration contains two
-benchmarks. The first benchmark name is "KMeans-1", and is executed on `KMeans`
-stage. The stage is benchmarked against 10000 randomly generated vectors.
+Combining the format requirements above, an example configuration is as follows.
+This configuration contains two benchmarks. The first benchmark name is
+"KMeans-1", and is executed on `KMeans` stage. The stage is benchmarked against
+10000 randomly generated vectors.
```json
{
@@ -194,3 +224,20 @@ stage. The stage is benchmarked against 10000 randomly generated vectors.
}
}
```
+
+### Benchmark Results Visualization
+
+`benchmark-results-visualize.py` is provided as a helper script to visualize
+benchmark results. For example, we can visualize the benchmark results generated
+in `results.json` as follows.
+
+```shell
+python3 ./bin/benchmark-results-visualize.py results.json --pattern "^KMeansModel.*$"
+```
+
+This command selects all benchmark results whose names start with `KmeansModel`
+and visualizes their `inputThroughput` against `numValues` as below. It is clear
+from this plot that in our example environment, when number of data is small,
+the algorithm's throughput is positively correlated to the number of data.
+
+
diff --git a/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/Benchmark.java b/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/Benchmark.java
index f870dc6..caeb602 100644
--- a/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/Benchmark.java
+++ b/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/Benchmark.java
@@ -31,86 +31,96 @@ import org.apache.commons.cli.Options;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
+import java.util.Collections;
import java.util.Map;
/** Entry class for benchmark execution. */
public class Benchmark {
- private static final Logger LOG = LoggerFactory.getLogger(Benchmark.class);
-
static final String VERSION_KEY = "version";
- static final Option HELP_OPTION =
+ private static final Logger LOG = LoggerFactory.getLogger(Benchmark.class);
+
+ private static final Option HELP_OPTION =
Option.builder("h")
.longOpt("help")
.desc("Show the help message for the command line interface.")
.build();
- static final Option OUTPUT_FILE_OPTION =
+ private static final Option OUTPUT_FILE_OPTION =
Option.builder()
.longOpt("output-file")
.desc("The output file name to save benchmark results.")
.hasArg()
.build();
- static final Options OPTIONS =
+ private static final Options OPTIONS =
new Options().addOption(HELP_OPTION).addOption(OUTPUT_FILE_OPTION);
- public static void printHelp() {
+ private static void printHelp() {
HelpFormatter formatter = new HelpFormatter();
formatter.setLeftPadding(5);
formatter.setWidth(80);
- System.out.println("./flink-ml-benchmark.sh <config-file-path> [OPTIONS]");
- System.out.println();
+ System.out.println("./benchmark-run.sh <config-file-path> [OPTIONS]\n");
formatter.setSyntaxPrefix("The following options are available:");
formatter.printHelp(" ", OPTIONS);
System.out.println();
}
- @SuppressWarnings("unchecked")
- public static void executeBenchmarks(CommandLine commandLine) throws Exception {
+ private static void executeBenchmarks(CommandLine commandLine) throws Exception {
String configFile = commandLine.getArgs()[0];
- Map<String, ?> benchmarks = BenchmarkUtils.parseJsonFile(configFile);
- System.out.println("Found benchmarks " + benchmarks.keySet());
+ Map<String, Map<String, Map<String, ?>>> benchmarks =
+ BenchmarkUtils.parseJsonFile(configFile);
+ System.out.printf("Found %d benchmarks.\n", benchmarks.keySet().size());
+ String saveFile = commandLine.getOptionValue(OUTPUT_FILE_OPTION.getLongOpt());
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
- List<BenchmarkResult> results = new ArrayList<>();
- for (Map.Entry<String, ?> benchmark : benchmarks.entrySet()) {
- LOG.info("Running benchmark " + benchmark.getKey() + ".");
-
- BenchmarkResult result =
- BenchmarkUtils.runBenchmark(
- tEnv, benchmark.getKey(), (Map<String, ?>) benchmark.getValue());
-
- results.add(result);
- LOG.info(BenchmarkUtils.getResultsMapAsJson(result));
+ int index = 0;
+ for (Map.Entry<String, Map<String, Map<String, ?>>> entry : benchmarks.entrySet()) {
+ String benchmarkName = entry.getKey();
+ Map<String, Map<String, ?>> benchmarkMap = entry.getValue();
+
+ LOG.info(
+ String.format(
+ "Running benchmark %d/%d: %s",
+ index++, benchmarks.keySet().size(), benchmarkName));
+
+ try {
+ BenchmarkResult result =
+ BenchmarkUtils.runBenchmark(tEnv, benchmarkName, benchmarkMap);
+ benchmarkMap.put("results", result.toMap());
+ LOG.info(String.format("Benchmark %s finished.\n%s", benchmarkName, benchmarkMap));
+ } catch (Exception e) {
+ benchmarkMap.put(
+ "results",
+ Collections.singletonMap(
+ "exception",
+ String.format(
+ "%s(%s:%s)",
+ e,
+ e.getStackTrace()[0].getFileName(),
+ e.getStackTrace()[0].getLineNumber())));
+ LOG.error(String.format("Benchmark %s failed.\n%s", benchmarkName, e));
+ }
}
-
- String benchmarkResultsJson =
- BenchmarkUtils.getResultsMapAsJson(results.toArray(new BenchmarkResult[0]));
System.out.println("Benchmarks execution completed.");
- System.out.println("Benchmark results summary:");
- System.out.println(benchmarkResultsJson);
+ String benchmarkResultsJson =
+ ReadWriteUtils.OBJECT_MAPPER
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(benchmarks);
if (commandLine.hasOption(OUTPUT_FILE_OPTION.getLongOpt())) {
- String saveFile = commandLine.getOptionValue(OUTPUT_FILE_OPTION.getLongOpt());
ReadWriteUtils.saveToFile(saveFile, benchmarkResultsJson, true);
- System.out.println("Benchmark results saved as json in " + saveFile + ".");
+ System.out.printf("Benchmark results saved as json in %s.\n", saveFile);
+ } else {
+ System.out.printf("Benchmark results summary:\n%s\n", benchmarkResultsJson);
}
}
- public static void printInvalidError(String[] args) {
- System.out.println("Invalid command line arguments " + Arrays.toString(args));
- System.out.println();
- System.out.println("Specify the help option (-h or --help) to get help on the command.");
- }
-
public static void main(String[] args) throws Exception {
CommandLineParser parser = new DefaultParser();
CommandLine commandLine = parser.parse(OPTIONS, args);
@@ -120,7 +130,9 @@ public class Benchmark {
} else if (commandLine.getArgs().length == 1) {
executeBenchmarks(commandLine);
} else {
- printInvalidError(args);
+ System.out.printf("Invalid command line arguments %s\n\n", Arrays.toString(args));
+ System.out.println(
+ "Specify the help option (-h or --help) to get help on the command.");
}
}
}
diff --git a/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkResult.java b/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkResult.java
index 4efdec0..4d57ed4 100644
--- a/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkResult.java
+++ b/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkResult.java
@@ -20,6 +20,9 @@ package org.apache.flink.ml.benchmark;
import org.apache.flink.util.Preconditions;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
/** The result of executing a benchmark. */
public class BenchmarkResult {
/** The benchmark name. */
@@ -55,4 +58,16 @@ public class BenchmarkResult {
this.outputRecordNum = outputRecordNum;
this.outputThroughput = outputThroughput;
}
+
+ public Map<String, ?> toMap() {
+ return new LinkedHashMap<String, Object>() {
+ {
+ put("totalTimeMs", totalTimeMs);
+ put("inputRecordNum", inputRecordNum);
+ put("inputThroughput", inputThroughput);
+ put("outputRecordNum", outputRecordNum);
+ put("outputThroughput", outputThroughput);
+ }
+ };
+ }
}
diff --git a/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkUtils.java b/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkUtils.java
index f21b43b..9a88153 100644
--- a/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkUtils.java
+++ b/flink-ml-benchmark/src/main/java/org/apache/flink/ml/benchmark/BenchmarkUtils.java
@@ -35,32 +35,33 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.util.Preconditions;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
-
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
+import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
/** Utility methods for benchmarks. */
public class BenchmarkUtils {
/** Loads benchmark configuration maps from the provided json file. */
@SuppressWarnings("unchecked")
- public static Map<String, ?> parseJsonFile(String path) throws IOException {
+ public static Map<String, Map<String, Map<String, ?>>> parseJsonFile(String path)
+ throws IOException {
InputStream inputStream = new FileInputStream(path);
Map<String, ?> jsonMap = ReadWriteUtils.OBJECT_MAPPER.readValue(inputStream, Map.class);
Preconditions.checkArgument(
jsonMap.containsKey(Benchmark.VERSION_KEY)
&& jsonMap.get(Benchmark.VERSION_KEY).equals(1));
- return jsonMap.entrySet().stream()
- .filter(x -> !x.getKey().equals(Benchmark.VERSION_KEY))
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ Map<String, Map<String, Map<String, ?>>> result = new HashMap<>();
+ for (Map.Entry<String, ?> entry : jsonMap.entrySet()) {
+ if (entry.getKey().equals(Benchmark.VERSION_KEY)) {
+ continue;
+ }
+ result.put(entry.getKey(), (Map<String, Map<String, ?>>) entry.getValue());
+ }
+ return result;
}
/**
@@ -71,14 +72,14 @@ public class BenchmarkUtils {
*/
@SuppressWarnings({"unchecked", "rawtypes"})
public static BenchmarkResult runBenchmark(
- StreamTableEnvironment tEnv, String name, Map<String, ?> params) throws Exception {
- Stage stage = ReadWriteUtils.instantiateWithParams((Map<String, ?>) params.get("stage"));
+ StreamTableEnvironment tEnv, String name, Map<String, Map<String, ?>> params)
+ throws Exception {
+ Stage stage = ReadWriteUtils.instantiateWithParams(params.get("stage"));
InputDataGenerator inputDataGenerator =
- ReadWriteUtils.instantiateWithParams((Map<String, ?>) params.get("inputData"));
+ ReadWriteUtils.instantiateWithParams(params.get("inputData"));
DataGenerator modelDataGenerator = null;
if (params.containsKey("modelData")) {
- modelDataGenerator =
- ReadWriteUtils.instantiateWithParams((Map<String, ?>) params.get("modelData"));
+ modelDataGenerator = ReadWriteUtils.instantiateWithParams(params.get("modelData"));
}
return runBenchmark(tEnv, name, stage, inputDataGenerator, modelDataGenerator);
@@ -135,25 +136,6 @@ public class BenchmarkUtils {
outputThroughput);
}
- /** Converts the benchmark results to a json string as a map. */
- public static String getResultsMapAsJson(BenchmarkResult... results)
- throws JsonProcessingException {
- List<Map<String, ?>> resultsMap = new ArrayList<>();
- for (BenchmarkResult result : results) {
- Map<String, Object> map = new LinkedHashMap<>();
- map.put("name", result.name);
- map.put("totalTimeMs", result.totalTimeMs);
- map.put("inputRecordNum", result.inputRecordNum);
- map.put("inputThroughput", result.inputThroughput);
- map.put("outputRecordNum", result.outputRecordNum);
- map.put("outputThroughput", result.outputThroughput);
- resultsMap.add(map);
- }
- return ReadWriteUtils.OBJECT_MAPPER
- .writerWithDefaultPrettyPrinter()
- .writeValueAsString(resultsMap);
- }
-
/**
* A stream sink that counts the number of all elements. The counting result is stored in an
* {@link org.apache.flink.api.common.accumulators.Accumulator} specified by {@link
diff --git a/flink-ml-benchmark/src/main/resources/benchmark-conf.json b/flink-ml-benchmark/src/main/resources/benchmark-conf.json
index 280453e..af246b4 100644
--- a/flink-ml-benchmark/src/main/resources/benchmark-conf.json
+++ b/flink-ml-benchmark/src/main/resources/benchmark-conf.json
@@ -45,5 +45,154 @@
"vectorDim": 10
}
}
+ },
+ "KMeansModel-2": {
+ "stage": {
+ "className": "org.apache.flink.ml.clustering.kmeans.KMeansModel",
+ "paramMap": {
+ "featuresCol": "features",
+ "k": 2,
+ "distanceMeasure": "euclidean",
+ "predictionCol": "prediction"
+ }
+ },
+ "modelData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.clustering.KMeansModelDataGenerator",
+ "paramMap": {
+ "seed": 1,
+ "arraySize": 2,
+ "vectorDim": 10
+ }
+ },
+ "inputData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap": {
+ "seed": 2,
+ "colNames": ["features"],
+ "numValues": 20000,
+ "vectorDim": 10
+ }
+ }
+ },
+ "KMeansModel-3": {
+ "stage": {
+ "className": "org.apache.flink.ml.clustering.kmeans.KMeansModel",
+ "paramMap": {
+ "featuresCol": "features",
+ "k": 2,
+ "distanceMeasure": "euclidean",
+ "predictionCol": "prediction"
+ }
+ },
+ "modelData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.clustering.KMeansModelDataGenerator",
+ "paramMap": {
+ "seed": 1,
+ "arraySize": 2,
+ "vectorDim": 10
+ }
+ },
+ "inputData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap": {
+ "seed": 2,
+ "colNames": ["features"],
+ "numValues": 30000,
+ "vectorDim": 10
+ }
+ }
+ },
+ "KMeansModel-4": {
+ "stage": {
+ "className": "org.apache.flink.ml.clustering.kmeans.KMeansModel",
+ "paramMap": {
+ "featuresCol": "features",
+ "k": 2,
+ "distanceMeasure": "euclidean",
+ "predictionCol": "prediction"
+ }
+ },
+ "modelData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.clustering.KMeansModelDataGenerator",
+ "paramMap": {
+ "seed": 1,
+ "arraySize": 2,
+ "vectorDim": 10
+ }
+ },
+ "inputData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap": {
+ "seed": 2,
+ "colNames": ["features"],
+ "numValues": 40000,
+ "vectorDim": 10
+ }
+ }
+ },
+ "KMeansModel-5": {
+ "stage": {
+ "className": "org.apache.flink.ml.clustering.kmeans.KMeansModel",
+ "paramMap": {
+ "featuresCol": "features",
+ "k": 2,
+ "distanceMeasure": "euclidean",
+ "predictionCol": "prediction"
+ }
+ },
+ "modelData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.clustering.KMeansModelDataGenerator",
+ "paramMap": {
+ "seed": 1,
+ "arraySize": 2,
+ "vectorDim": 10
+ }
+ },
+ "inputData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap": {
+ "seed": 2,
+ "colNames": ["features"],
+ "numValues": 50000,
+ "vectorDim": 10
+ }
+ }
+ },
+ "Undefined-Parameter": {
+ "stage": {
+ "className": "org.apache.flink.ml.clustering.kmeans.KMeans",
+ "paramMap": {
+ "featuresCol": "features",
+ "featureCol": "feature",
+ "predictionCol": "prediction"
+ }
+ },
+ "inputData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap": {
+ "seed": 2,
+ "colNames": ["features"],
+ "numValues": 10000,
+ "vectorDim": 10
+ }
+ }
+ },
+ "Unmatch-Input": {
+ "stage": {
+ "className": "org.apache.flink.ml.clustering.kmeans.KMeans",
+ "paramMap": {
+ "featuresCol": "features",
+ "predictionCol": "prediction"
+ }
+ },
+ "inputData": {
+ "className": "org.apache.flink.ml.benchmark.datagenerator.common.DenseVectorGenerator",
+ "paramMap": {
+ "seed": 2,
+ "colNames": ["non-features"],
+ "numValues": 10000,
+ "vectorDim": 10
+ }
+ }
}
}
diff --git a/flink-ml-benchmark/src/test/java/org/apache/flink/ml/benchmark/BenchmarkTest.java b/flink-ml-benchmark/src/test/java/org/apache/flink/ml/benchmark/BenchmarkTest.java
index c3ca0cf..eecc6f0 100644
--- a/flink-ml-benchmark/src/test/java/org/apache/flink/ml/benchmark/BenchmarkTest.java
+++ b/flink-ml-benchmark/src/test/java/org/apache/flink/ml/benchmark/BenchmarkTest.java
@@ -49,7 +49,7 @@ public class BenchmarkTest extends AbstractTestBase {
FileUtils.copyInputStreamToFile(inputStream, configFile);
Map<String, ?> benchmarks = BenchmarkUtils.parseJsonFile(configFile.getAbsolutePath());
- assertEquals(benchmarks.size(), 2);
+ assertEquals(benchmarks.size(), 8);
assertTrue(benchmarks.containsKey("KMeans-1"));
assertTrue(benchmarks.containsKey("KMeansModel-1"));
}
@@ -59,7 +59,7 @@ public class BenchmarkTest extends AbstractTestBase {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
- Map<String, Object> params = new HashMap<>();
+ Map<String, Map<String, ?>> params = new HashMap<>();
Map<String, Object> stageParams = new HashMap<>();
stageParams.put("className", KMeans.class.getCanonicalName());
diff --git a/flink-ml-core/src/main/java/org/apache/flink/ml/util/ReadWriteUtils.java b/flink-ml-core/src/main/java/org/apache/flink/ml/util/ReadWriteUtils.java
index ed7b227..35b3e7e 100644
--- a/flink-ml-core/src/main/java/org/apache/flink/ml/util/ReadWriteUtils.java
+++ b/flink-ml-core/src/main/java/org/apache/flink/ml/util/ReadWriteUtils.java
@@ -71,8 +71,8 @@ public class ReadWriteUtils {
return param.jsonEncode((T) value);
}
- // Converts Map<Param<?>, Object> to Map<String, String> which maps the parameter name to the
- // string-encoded parameter value.
+ // Converts Map<Param<?>, Object> to Map<String, Object> which maps the parameter name to the
+ // json-supported parameter value.
private static Map<String, Object> jsonEncode(Map<Param<?>, Object> paramMap)
throws IOException {
Map<String, Object> result = new HashMap<>(paramMap.size());
diff --git a/flink-ml-dist/src/main/flink-ml-bin/bin/benchmark-results-visualize.py b/flink-ml-dist/src/main/flink-ml-bin/bin/benchmark-results-visualize.py
new file mode 100644
index 0000000..66fc047
--- /dev/null
+++ b/flink-ml-dist/src/main/flink-ml-bin/bin/benchmark-results-visualize.py
@@ -0,0 +1,68 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import argparse
+import json
+import matplotlib.pyplot as plt
+import re
+
+
+def get_nested_field_value(nested_fields, field_names):
+ for field_name in field_names:
+ nested_fields = nested_fields[field_name]
+ return nested_fields
+
+
+def visualize_benchmark_results(file_name, name_pattern, x_field, y_field):
+ x_values = []
+ y_values = []
+ with open(file_name, "r") as config_file:
+ for name, config in json.loads(config_file.read()).items():
+ if not name_pattern.match(name):
+ continue
+ x_values.append(get_nested_field_value(config, x_field.split(".")))
+ y_values.append(get_nested_field_value(config, y_field.split(".")))
+
+ plt.scatter(x_values, y_values)
+ plt.xlabel(x_field)
+ plt.ylabel(y_field)
+ plt.title("Flink ML Benchmark Results Visualization")
+ plt.show()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description="Visualizes benchmark results.")
+ parser.add_argument(
+ "file_name", help="Json file to acquire benchmark results.")
+ parser.add_argument(
+ "--pattern", help="Regex pattern of benchmark names. "
+ "Benchmarks whose names match this pattern will be selected for visualization. "
+ "If not set, all benchmarks will be selected.",
+ default=".*")
+
+ field_help_grammar = "Name of the {} field. Use dot(.) to identify nested fields. Default value: `{}`."
+ parser.add_argument(
+ "--x-field", help=field_help_grammar.format("independent", "inputData.paramMap.numValues"),
+ default="inputData.paramMap.numValues")
+ parser.add_argument(
+ "--y-field", help=field_help_grammar.format("dependent", "results.inputThroughput"),
+ default="results.inputThroughput")
+ args = parser.parse_args()
+
+ visualize_benchmark_results(args.file_name, re.compile(
+ args.pattern), args.x_field, args.y_field)
diff --git a/flink-ml-dist/src/main/flink-ml-bin/bin/flink-ml-benchmark.sh b/flink-ml-dist/src/main/flink-ml-bin/bin/benchmark-run.sh
similarity index 100%
rename from flink-ml-dist/src/main/flink-ml-bin/bin/flink-ml-benchmark.sh
rename to flink-ml-dist/src/main/flink-ml-bin/bin/benchmark-run.sh