You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by an...@apache.org on 2023/09/29 19:52:26 UTC

[beam] 01/01: Add GBK and combine python batch load tests to perf alert tool

This is an automated email from the ASF dual-hosted git repository.

anandinguva pushed a commit to branch onboard_python_tests
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 744ad3ccb0c2afa2e62782ac9484a7cd45b80824
Author: Anand Inguva <an...@gmail.com>
AuthorDate: Fri Sep 29 15:52:12 2023 -0400

    Add GBK and combine python batch load tests to perf alert tool
---
 .../testing/analyzers/tests_config.yaml            | 222 +++++++++++++++------
 1 file changed, 160 insertions(+), 62 deletions(-)

diff --git a/sdks/python/apache_beam/testing/analyzers/tests_config.yaml b/sdks/python/apache_beam/testing/analyzers/tests_config.yaml
index bc74f292c48..0350b7be439 100644
--- a/sdks/python/apache_beam/testing/analyzers/tests_config.yaml
+++ b/sdks/python/apache_beam/testing/analyzers/tests_config.yaml
@@ -18,86 +18,184 @@
 # for the unique key to define a test, please use the following format:
 # {test_name}-{metric_name}
 
-pytorch_image_classification_benchmarks-resnet152-mean_inference_batch_latency_micro_secs:
+# pytorch_image_classification_benchmarks-resnet152-mean_inference_batch_latency_micro_secs:
+#   test_description:
+#     Pytorch image classification on 50k images of size 224 x 224 with resnet 152.
+#     Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L63
+#     Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2
+#   test_target:
+#   metrics_dataset: beam_run_inference
+#   metrics_table: torch_inference_imagenet_results_resnet152
+#   project: apache-beam-testing
+#   metric_name: mean_inference_batch_latency_micro_secs
+
+# pytorch_image_classification_benchmarks-resnet101-mean_load_model_latency_milli_secs:
+#   test_description:
+#     Pytorch image classification on 50k images of size 224 x 224 with resnet 101.
+#     Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34
+#     Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
+#   test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+#   metrics_dataset: beam_run_inference
+#   metrics_table: torch_inference_imagenet_results_resnet101
+#   project: apache-beam-testing
+#   metric_name: mean_load_model_latency_milli_secs
+
+# pytorch_image_classification_benchmarks-resnet101-mean_inference_batch_latency_micro_secs:
+#   test_description:
+#     Pytorch image classification on 50k images of size 224 x 224 with resnet 101.
+#     Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34
+#     Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2
+#   test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+#   metrics_dataset: beam_run_inference
+#   metrics_table: torch_inference_imagenet_results_resnet101
+#   project: apache-beam-testing
+#   metric_name: mean_inference_batch_latency_micro_secs
+
+# pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs:
+#   test_description:
+#     Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU.
+#     Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151
+#     Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
+#   test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+#   metrics_dataset: beam_run_inference
+#   metrics_table: torch_inference_imagenet_results_resnet101
+#   project: apache-beam-testing
+#   metric_name: mean_inference_batch_latency_micro_secs
+
+# pytorch_image_classification_benchmarks-resnet152-GPU-mean_load_model_latency_milli_secs:
+#   test_description:
+#     Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU.
+#     Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151
+#     Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
+#   test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+#   metrics_dataset: beam_run_inference
+#   metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
+#   project: apache-beam-testing
+#   metric_name: mean_load_model_latency_milli_secs
+
+# pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs:
+#   test_description:
+#     Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU.
+#     Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151).
+#     Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?from=now-90d&to=now&viewPanel=2
+#   test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
+#   metrics_dataset: beam_run_inference
+#   metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
+#   project: apache-beam-testing
+#   metric_name: mean_inference_batch_latency_micro_secs
+
+# test_cloudml_benchmark_cirteo_no_shuffle_10GB-runtime_sec:
+#   test_description:
+#     TFT Criteo test on 10 GB data with no Reshuffle.
+#     Test link - [Test link](https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#L82)
+#   metrics_dataset: beam_cloudml
+#   metrics_table: cloudml_benchmark_cirteo_no_shuffle_10GB
+#   project: apache-beam-testing
+#   metric_name: runtime_sec
+
+# test_cloudml_benchmark_criteo_10GB-runtime_sec:
+#   test_description:
+#     TFT Criteo test on 10 GB data.
+#     Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#LL104C7-L104C41
+#   metrics_dataset: beam_cloudml
+#   metrics_table: cloudml_benchmark_criteo_10GB
+#   project: apache-beam-testing
+#   metric_name: runtime_sec
+
+
+
+# Python Combine load tests at http://35.193.202.176/d/WNzYt13Zk/combine-load-tests?orgId=1
+combine_python_batch_2gb_10_byte_records:
   test_description:
-    Pytorch image classification on 50k images of size 224 x 224 with resnet 152.
-    Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L63
-    Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2
-  test_target:
-  metrics_dataset: beam_run_inference
-  metrics_table: torch_inference_imagenet_results_resnet152
+    Combine Python Load Test 2 GB 10 byte records
+    Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy#L76C24-L76C65
+    Test dashboard - http://35.193.202.176/d/WNzYt13Zk/combine-load-tests?orgId=1&from=now-90d&to=now&var-processingType=batch&var-sdk=python&viewPanel=2
+  test_target: apache_beam.testing.load_tests.combine_test
+  metrics_dataset: load_test
+  metrics_table: python_dataflow_batch_combine_1
+  metric_name: runtime
   project: apache-beam-testing
-  metric_name: mean_inference_batch_latency_micro_secs
 
-pytorch_image_classification_benchmarks-resnet101-mean_load_model_latency_milli_secs:
+combine_python_batch_2gb_fanout_4:
   test_description:
-    Pytorch image classification on 50k images of size 224 x 224 with resnet 101.
-    Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34
-    Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
-  test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
-  metrics_dataset: beam_run_inference
-  metrics_table: torch_inference_imagenet_results_resnet101
+    Combine Python Load test - 2GB Fanout 4
+    Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_Combine_Python.groovy#L52
+    Test Dashboard - http://35.193.202.176/d/WNzYt13Zk/combine-load-tests?orgId=1&from=now-90d&to=now&var-processingType=batch&var-sdk=python&viewPanel=4
+  test_target: apache_beam.testing.load_tests.combine_test
+  metrics_dataset: load_test
+  metrics_table: python_dataflow_batch_combine_4
+  metric_name: runtime
   project: apache-beam-testing
-  metric_name: mean_load_model_latency_milli_secs
 
-pytorch_image_classification_benchmarks-resnet101-mean_inference_batch_latency_micro_secs:
-  test_description:
-    Pytorch image classification on 50k images of size 224 x 224 with resnet 101.
-    Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L34
-    Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=2
-  test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
-  metrics_dataset: beam_run_inference
-  metrics_table: torch_inference_imagenet_results_resnet101
+combine_python_batch_2gb_fanout_8:
+  test_description: Combine Python Load test - 2GB Fanout 8
+  test_target: apache_beam.testing.load_tests.combine_test
+  metrics_dataset: load_test
+  metrics_table: python_dataflow_batch_combine_5
+  metric_name: runtime
   project: apache-beam-testing
-  metric_name: mean_inference_batch_latency_micro_secs
 
-pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs:
+# Python Batch GBK load tests at http://35.193.202.176/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now
+gbk_python_batch_load_test_2gb_of_10B_records:
   test_description:
-    Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU.
-    Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151
-    Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
-  test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
-  metrics_dataset: beam_run_inference
-  metrics_table: torch_inference_imagenet_results_resnet101
+    GroupByKey Python Load test - 2GB of 10B records
+    python | GBK | Small records (10B)
+    Test Dashboard - http://35.193.202.176/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=2
+    Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L36C25-L36C72
+  test_target: apache_beam.testing.load_tests.group_by_key_test
+  metrics_table: python_dataflow_batch_gbk_1
+  metrics_dataset: load_test
+  metric_name: runtime
   project: apache-beam-testing
-  metric_name: mean_inference_batch_latency_micro_secs
 
-pytorch_image_classification_benchmarks-resnet152-GPU-mean_load_model_latency_milli_secs:
+gbk_python_batch_load_test_2gb_of_100B_records:
   test_description:
-    Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU.
-    Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151
-    Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?orgId=1&viewPanel=7
-  test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
-  metrics_dataset: beam_run_inference
-  metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
+    GroupByKey Python Load test - 2GB of 100B records
+    python | GBK | Medium records (100B)
+    Test Dashboard - http://35.193.202.176/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=3
+    Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L60
+  test_target: apache_beam.testing.load_tests.group_by_key_test
+  metrics_table: python_dataflow_batch_gbk_2
+  metrics_dataset: load_test
+  metric_name: runtime
   project: apache-beam-testing
-  metric_name: mean_load_model_latency_milli_secs
 
-pytorch_image_classification_benchmarks-resnet152-GPU-mean_inference_batch_latency_micro_secs:
+gbk_python_batch_load_test_2gb_of_100KB_records:
   test_description:
-    Pytorch image classification on 50k images of size 224 x 224 with resnet 152 with Tesla T4 GPU.
-    Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/.test-infra/jenkins/job_InferenceBenchmarkTests_Python.groovy#L151).
-    Test dashboard - http://metrics.beam.apache.org/d/ZpS8Uf44z/python-ml-runinference-benchmarks?from=now-90d&to=now&viewPanel=2
-  test_target: apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
-  metrics_dataset: beam_run_inference
-  metrics_table: torch_inference_imagenet_results_resnet152_tesla_t4
+    GroupByKey Python Load test - 2GB of 100kB records
+    python | GBK | Large records (100kB)
+    Test Dashboard - http://35.193.202.176/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-6M&to=now&viewPanel=4&inspect=4
+    Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L83
+  test_target: apache_beam.testing.load_tests.group_by_key_test
+  metrics_table: python_dataflow_batch_gbk_3
+  metrics_dataset: load_test
+  metric_name: runtime
   project: apache-beam-testing
-  metric_name: mean_inference_batch_latency_micro_secs
 
-test_cloudml_benchmark_cirteo_no_shuffle_10GB-runtime_sec:
+gbk_python_batch_load_test_fanout_4_times_witj_2GB_10byte_records_total:
+# this test looks little noisy. Remove this if it causes too many false
+# positives.
   test_description:
-    TFT Criteo test on 10 GB data with no Reshuffle.
-    Test link - [Test link](https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#L82)
-  metrics_dataset: beam_cloudml
-  metrics_table: cloudml_benchmark_cirteo_no_shuffle_10GB
+    GroupByKey Python Load test - fanout 4 times with 2GB 10-byte records total
+    python | GBK | Fanout 4
+    Test Dashboard - http://35.193.202.176/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=5
+    Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L106
+  test_target: apache_beam.testing.load_tests.group_by_key_test
+  metrics_table: python_dataflow_batch_gbk_4
+  metrics_dataset: load_test
+  metric_name: runtime
   project: apache-beam-testing
-  metric_name: runtime_sec
 
-test_cloudml_benchmark_criteo_10GB-runtime_sec:
+
+gbk_python_batch_load_test_fanout_8_times_witj_2GB_10byte_records_total:
+# this test looks little noisy. Remove this if it causes too many false
+# positives.
   test_description:
-    TFT Criteo test on 10 GB data.
-    Test link - https://github.com/apache/beam/blob/42d0a6e3564d8b9c5d912428a6de18fb22a13ac1/sdks/python/apache_beam/testing/benchmarks/cloudml/cloudml_benchmark_test.py#LL104C7-L104C41
-  metrics_dataset: beam_cloudml
-  metrics_table: cloudml_benchmark_criteo_10GB
-  project: apache-beam-testing
-  metric_name: runtime_sec
+    GroupByKey Python Load test - fanout 8 times with 2GB 10-byte records total
+    python | GBK | Fanout 8
+    Test Dashboard - http://35.193.202.176/d/UYZ-oJ3Zk/gbk-load-tests?orgId=1&var-processingType=batch&var-sdk=python&from=now-90d&to=now&viewPanel=6
+    Test link - https://github.com/apache/beam/blob/5e38decf9e723a385057131b01bbd33d8c60bda3/.test-infra/jenkins/job_LoadTests_GBK_Python.groovy#L129
+  metrics_table: python_dataflow_batch_gbk_5
+  metrics_dataset: load_test
+  metric_name: runtime
+  project: apache-beam-testing
\ No newline at end of file