You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jr...@apache.org on 2025/03/26 15:56:06 UTC
(beam) branch master updated: Update dataflow_cost_benchmark. Update website: add ML pipelines performance metrics (#34361)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0766593e0fb Update dataflow_cost_benchmark. Update website: add ML pipelines performance metrics (#34361)
0766593e0fb is described below
commit 0766593e0fbf74b6dfcadf22139a71ce361b4240
Author: Vitaly Terentyev <vi...@akvelon.com>
AuthorDate: Wed Mar 26 19:55:59 2025 +0400
Update dataflow_cost_benchmark. Update website: add ML pipelines performance metrics (#34361)
---
.../beam_Inference_Python_Benchmarks_Dataflow.yml | 2 +-
.../beam_Python_CostBenchmarks_Dataflow.yml | 1 +
.../python_wordcount.txt | 1 +
.test-infra/tools/refresh_looker_metrics.py | 6 +
.../ml/inference/tensorflow_tests_requirements.txt | 2 +-
.../ml/inference/torch_tests_requirements.txt | 1 +
.../pytorch_image_classification_benchmarks.py | 9 +-
.../pytorch_language_modeling_benchmarks.py | 4 +-
...nsorflow_mnist_classification_cost_benchmark.py | 2 +-
.../benchmarks/wordcount/requirements.txt} | 5 +-
.../testing/benchmarks/wordcount/wordcount.py | 2 +-
.../testing/load_tests/dataflow_cost_benchmark.py | 143 ++++++++++++++++++---
website/www/site/content/en/performance/_index.md | 15 ++-
.../en/performance/pytorchbertbase/_index.md | 34 +++++
.../en/performance/pytorchbertlarge/_index.md | 34 +++++
.../en/performance/pytorchresnet101/_index.md | 34 +++++
.../en/performance/pytorchresnet152/_index.md | 34 +++++
.../en/performance/pytorchresnet152tesla/_index.md | 34 +++++
.../en/performance/tensorflowmnist/_index.md | 34 +++++
website/www/site/data/performance.yaml | 96 ++++++++++++++
20 files changed, 463 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml
index 58c4de11e85..a18def996ac 100644
--- a/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml
+++ b/.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml
@@ -109,7 +109,7 @@ jobs:
-PpythonVersion=3.10 \
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-pytorch-imagenet-python-152-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152-${{env.NOW_UTC}}.txt' \
- - name: run Pytorch Language Modeling using Hugging face bert-base-uncased model
+ - name: run Pytorch Language Modeling using Hugging Face bert-base-uncased model
uses: ./.github/actions/gradle-command-self-hosted-action
timeout-minutes: 180
with:
diff --git a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml
index dbba0922f88..32999542251 100644
--- a/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml
+++ b/.github/workflows/beam_Python_CostBenchmarks_Dataflow.yml
@@ -92,6 +92,7 @@ jobs:
-PloadTest.mainClass=apache_beam.testing.benchmarks.wordcount.wordcount \
-Prunner=DataflowRunner \
-PpythonVersion=3.10 \
+ -PloadTest.requirementsTxtFile=apache_beam/testing/benchmarks/wordcount/requirements.txt \
'-PloadTest.args=${{ env.beam_Python_Cost_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-wordcount-python-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/wordcount/result_wordcount-${{env.NOW_UTC}}.txt' \
- name: Run Tensorflow MNIST Image Classification on Dataflow
uses: ./.github/actions/gradle-command-self-hosted-action
diff --git a/.github/workflows/cost-benchmarks-pipeline-options/python_wordcount.txt b/.github/workflows/cost-benchmarks-pipeline-options/python_wordcount.txt
index 424936ddad9..35239345183 100644
--- a/.github/workflows/cost-benchmarks-pipeline-options/python_wordcount.txt
+++ b/.github/workflows/cost-benchmarks-pipeline-options/python_wordcount.txt
@@ -22,6 +22,7 @@
--input_options={}
--staging_location=gs://temp-storage-for-perf-tests/loadtests
--temp_location=gs://temp-storage-for-perf-tests/loadtests
+--requirements_file=apache_beam/testing/benchmarks/wordcount/requirements.txt
--publish_to_big_query=true
--metrics_dataset=beam_run_inference
--metrics_table=python_wordcount
diff --git a/.test-infra/tools/refresh_looker_metrics.py b/.test-infra/tools/refresh_looker_metrics.py
index 842fdd6ac10..4ebbf9744ad 100644
--- a/.test-infra/tools/refresh_looker_metrics.py
+++ b/.test-infra/tools/refresh_looker_metrics.py
@@ -34,6 +34,12 @@ LOOKS_TO_DOWNLOAD = [
("33", ["21", "70", "116", "69", "115"]), # BigTableIO_Write
("34", ["22", "56", "96", "55", "95"]), # TextIO_Read
("35", ["23", "64", "110", "63", "109"]), # TextIO_Write
+ ("75", ["258", "259", "260", "261", "262"]), # TensorFlow MNIST
+ ("76", ["233", "234", "235", "236", "237"]), # PyTorch BERT base uncased
+ ("77", ["238", "239", "240", "241", "242"]), # PyTorch BERT large uncased
+ ("78", ["243", "244", "245", "246", "247"]), # PyTorch Resnet 101
+ ("79", ["248", "249", "250", "251", "252"]), # PyTorch Resnet 152
+ ("80", ["253", "254", "255", "256", "257"]), # PyTorch Resnet 152 Tesla T4
]
diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt b/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt
index e0a5c704de4..bc2113b5395 100644
--- a/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt
+++ b/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt
@@ -20,4 +20,4 @@ tensorflow>=2.12.0
tensorflow_hub>=0.10.0
Pillow>=9.0.0
typing-extensions>=4.8.0
-
+google-cloud-monitoring>=2.27.0
diff --git a/sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt b/sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt
index 790f015f9b2..df6273038f1 100644
--- a/sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt
+++ b/sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt
@@ -19,3 +19,4 @@ torch>=1.7.1
torchvision>=0.8.2
pillow>=8.0.0
transformers>=4.18.0
+google-cloud-monitoring>=2.27.0
\ No newline at end of file
diff --git a/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py b/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py
index 514c9d67285..a90c268ed53 100644
--- a/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py
+++ b/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py
@@ -19,19 +19,22 @@
import logging
from apache_beam.examples.inference import pytorch_image_classification
-from apache_beam.testing.load_tests.load_test import LoadTest
+from apache_beam.testing.load_tests.dataflow_cost_benchmark import DataflowCostBenchmark
from torchvision import models
_PERF_TEST_MODELS = ['resnet50', 'resnet101', 'resnet152']
_PRETRAINED_MODEL_MODULE = 'torchvision.models'
-class PytorchVisionBenchmarkTest(LoadTest):
+class PytorchVisionBenchmarkTest(DataflowCostBenchmark):
def __init__(self):
# TODO (https://github.com/apache/beam/issues/23008)
# make get_namespace() method in RunInference static
self.metrics_namespace = 'BeamML_PyTorch'
- super().__init__(metrics_namespace=self.metrics_namespace)
+ super().__init__(
+ metrics_namespace=self.metrics_namespace,
+ pcollection='PyTorchRunInference/BeamML_RunInference_Postprocess-0.out0'
+ )
def test(self):
pretrained_model_name = self.pipeline.get_option('pretrained_model_name')
diff --git a/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py b/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py
index 1d6ecb2bd43..282a7a4e35f 100644
--- a/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py
+++ b/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py
@@ -19,10 +19,10 @@
import logging
from apache_beam.examples.inference import pytorch_language_modeling
-from apache_beam.testing.load_tests.load_test import LoadTest
+from apache_beam.testing.load_tests.dataflow_cost_benchmark import DataflowCostBenchmark
-class PytorchLanguageModelingBenchmarkTest(LoadTest):
+class PytorchLanguageModelingBenchmarkTest(DataflowCostBenchmark):
def __init__(self):
# TODO (https://github.com/apache/beam/issues/23008):
# make get_namespace() method in RunInference static
diff --git a/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py b/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py
index 223b973e5fb..89750a3a1bd 100644
--- a/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py
+++ b/sdks/python/apache_beam/testing/benchmarks/inference/tensorflow_mnist_classification_cost_benchmark.py
@@ -24,7 +24,7 @@ from apache_beam.testing.load_tests.dataflow_cost_benchmark import DataflowCostB
class TensorflowMNISTClassificationCostBenchmark(DataflowCostBenchmark):
def __init__(self):
- super().__init__()
+ super().__init__(pcollection='PostProcessOutputs.out0')
def test(self):
extra_opts = {}
diff --git a/sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt b/sdks/python/apache_beam/testing/benchmarks/wordcount/requirements.txt
similarity index 92%
copy from sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt
copy to sdks/python/apache_beam/testing/benchmarks/wordcount/requirements.txt
index 790f015f9b2..19c4367ea3a 100644
--- a/sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt
+++ b/sdks/python/apache_beam/testing/benchmarks/wordcount/requirements.txt
@@ -15,7 +15,4 @@
# limitations under the License.
#
-torch>=1.7.1
-torchvision>=0.8.2
-pillow>=8.0.0
-transformers>=4.18.0
+google-cloud-monitoring>=2.27.0
\ No newline at end of file
diff --git a/sdks/python/apache_beam/testing/benchmarks/wordcount/wordcount.py b/sdks/python/apache_beam/testing/benchmarks/wordcount/wordcount.py
index 513ede47e80..73662512f57 100644
--- a/sdks/python/apache_beam/testing/benchmarks/wordcount/wordcount.py
+++ b/sdks/python/apache_beam/testing/benchmarks/wordcount/wordcount.py
@@ -24,7 +24,7 @@ from apache_beam.testing.load_tests.dataflow_cost_benchmark import DataflowCostB
class WordcountCostBenchmark(DataflowCostBenchmark):
def __init__(self):
- super().__init__()
+ super().__init__(pcollection='Format.out0')
def test(self):
extra_opts = {}
diff --git a/sdks/python/apache_beam/testing/load_tests/dataflow_cost_benchmark.py b/sdks/python/apache_beam/testing/load_tests/dataflow_cost_benchmark.py
index 96a1cd31e29..87af2ef6a50 100644
--- a/sdks/python/apache_beam/testing/load_tests/dataflow_cost_benchmark.py
+++ b/sdks/python/apache_beam/testing/load_tests/dataflow_cost_benchmark.py
@@ -17,13 +17,19 @@
# pytype: skip-file
import logging
+import re
import time
+from datetime import datetime
from typing import Any
from typing import Optional
+from google.cloud import monitoring_v3
+from google.protobuf.duration_pb2 import Duration
+
import apache_beam.testing.load_tests.dataflow_cost_consts as costs
from apache_beam.metrics.execution import MetricResult
from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult
+from apache_beam.runners.dataflow.internal.apiclient import DataflowApplicationClient
from apache_beam.runners.runner import PipelineState
from apache_beam.testing.load_tests.load_test import LoadTest
@@ -44,61 +50,88 @@ class DataflowCostBenchmark(LoadTest):
calculate the cost of the job later, as different accelerators have different
billing rates per hour of use.
"""
+
+ WORKER_START_PATTERN = re.compile(
+ r'^All workers have finished the startup processes and '
+ r'began to receive work requests.*$')
+ WORKER_STOP_PATTERN = re.compile(r'^Stopping worker pool.*$')
+
def __init__(
self,
metrics_namespace: Optional[str] = None,
is_streaming: bool = False,
- gpu: Optional[costs.Accelerator] = None):
+ gpu: Optional[costs.Accelerator] = None,
+ pcollection: str = 'ProcessOutput.out0'):
+ """
+ Initializes DataflowCostBenchmark.
+
+ Args:
+ metrics_namespace (Optional[str]): Namespace for metrics.
+ is_streaming (bool): Whether the pipeline is streaming or batch.
+ gpu (Optional[costs.Accelerator]): Optional GPU type.
+ pcollection (str): PCollection name to monitor throughput.
+ """
self.is_streaming = is_streaming
self.gpu = gpu
+ self.pcollection = pcollection
super().__init__(metrics_namespace=metrics_namespace)
+ self.dataflow_client = DataflowApplicationClient(
+ self.pipeline.get_pipeline_options())
+ self.monitoring_client = monitoring_v3.MetricServiceClient()
- def run(self):
+ def run(self) -> None:
try:
self.test()
if not hasattr(self, 'result'):
self.result = self.pipeline.run()
- # Defaults to waiting forever unless timeout has been set
state = self.result.wait_until_finish(duration=self.timeout_ms)
assert state != PipelineState.FAILED
+
logging.info(
'Pipeline complete, sleeping for 4 minutes to allow resource '
'metrics to populate.')
time.sleep(240)
+
self.extra_metrics = self._retrieve_cost_metrics(self.result)
+ additional_metrics = self._get_additional_metrics(self.result)
+ self.extra_metrics.update(additional_metrics)
+
+ logging.info(self.extra_metrics)
self._metrics_monitor.publish_metrics(self.result, self.extra_metrics)
finally:
self.cleanup()
def _retrieve_cost_metrics(self,
result: DataflowPipelineResult) -> dict[str, Any]:
+ """Calculates estimated cost based on pipeline resource usage."""
job_id = result.job_id()
metrics = result.metrics().all_metrics(job_id)
metrics_dict = self._process_metrics_list(metrics)
- logging.info(metrics_dict)
+
cost = 0.0
- if (self.is_streaming):
+ if self.is_streaming:
cost += metrics_dict.get(
"TotalVcpuTime", 0.0) / 3600 * costs.VCPU_PER_HR_STREAMING
- cost += (
- metrics_dict.get("TotalMemoryUsage", 0.0) /
- 1000) / 3600 * costs.MEM_PER_GB_HR_STREAMING
+ cost += metrics_dict.get(
+ "TotalMemoryUsage", 0.0) / 1000 / 3600 * costs.MEM_PER_GB_HR_STREAMING
cost += metrics_dict.get(
"TotalStreamingDataProcessed", 0.0) * costs.SHUFFLE_PER_GB_STREAMING
else:
cost += metrics_dict.get(
"TotalVcpuTime", 0.0) / 3600 * costs.VCPU_PER_HR_BATCH
- cost += (
- metrics_dict.get("TotalMemoryUsage", 0.0) /
- 1000) / 3600 * costs.MEM_PER_GB_HR_BATCH
+ cost += metrics_dict.get(
+ "TotalMemoryUsage", 0.0) / 1000 / 3600 * costs.MEM_PER_GB_HR_BATCH
cost += metrics_dict.get(
"TotalStreamingDataProcessed", 0.0) * costs.SHUFFLE_PER_GB_BATCH
- if (self.gpu):
+
+ if self.gpu:
rate = costs.ACCELERATOR_TO_COST[self.gpu]
cost += metrics_dict.get("TotalGpuTime", 0.0) / 3600 * rate
+
cost += metrics_dict.get("TotalPdUsage", 0.0) / 3600 * costs.PD_PER_GB_HR
cost += metrics_dict.get(
"TotalSsdUsage", 0.0) / 3600 * costs.PD_SSD_PER_GB_HR
+
metrics_dict["EstimatedCost"] = cost
return metrics_dict
@@ -109,7 +142,87 @@ class DataflowCostBenchmark(LoadTest):
metric_key = entry.key
metric = metric_key.metric
if metric_key.step == '' and metric.namespace == 'dataflow/v1b3':
- if entry.committed is None:
- entry.committed = 0.0
- system_metrics[metric.name] = entry.committed
+ system_metrics[metric.name] = entry.committed or 0.0
return system_metrics
+
+ def _get_worker_time_interval(
+ self, job_id: str) -> tuple[Optional[str], Optional[str]]:
+ """Extracts worker start and stop times from job messages."""
+ messages, _ = self.dataflow_client.list_messages(
+ job_id=job_id,
+ start_time=None,
+ end_time=None,
+ minimum_importance='JOB_MESSAGE_DETAILED')
+
+ start_time, end_time = None, None
+ for message in messages:
+ text = message.messageText
+ if text:
+ if self.WORKER_START_PATTERN.match(text):
+ start_time = message.time
+ if self.WORKER_STOP_PATTERN.match(text):
+ end_time = message.time
+
+ return start_time, end_time
+
+ def _get_throughput_metrics(
+ self, project: str, job_id: str, start_time: str,
+ end_time: str) -> dict[str, float]:
+ interval = monitoring_v3.TimeInterval(
+ start_time=start_time, end_time=end_time)
+ aggregation = monitoring_v3.Aggregation(
+ alignment_period=Duration(seconds=60),
+ per_series_aligner=monitoring_v3.Aggregation.Aligner.ALIGN_MEAN)
+
+ requests = {
+ "Bytes": monitoring_v3.ListTimeSeriesRequest(
+ name=f"projects/{project}",
+ filter=f'metric.type='
+ f'"dataflow.googleapis.com/job/estimated_bytes_produced_count" '
+ f'AND metric.labels.job_id='
+ f'"{job_id}" AND metric.labels.pcollection="{self.pcollection}"',
+ interval=interval,
+ aggregation=aggregation),
+ "Elements": monitoring_v3.ListTimeSeriesRequest(
+ name=f"projects/{project}",
+ filter=f'metric.type="dataflow.googleapis.com/job/element_count" '
+ f'AND metric.labels.job_id="{job_id}" '
+ f'AND metric.labels.pcollection="{self.pcollection}"',
+ interval=interval,
+ aggregation=aggregation)
+ }
+
+ metrics = {}
+ for key, req in requests.items():
+ time_series = self.monitoring_client.list_time_series(request=req)
+ values = [
+ point.value.double_value for series in time_series
+ for point in series.points
+ ]
+ metrics[f"AvgThroughput{key}"] = sum(values) / len(
+ values) if values else 0.0
+
+ return metrics
+
+ def _get_job_runtime(self, start_time: str, end_time: str) -> float:
+ """Calculates the job runtime duration in seconds."""
+ start_dt = datetime.fromisoformat(start_time[:-1])
+ end_dt = datetime.fromisoformat(end_time[:-1])
+ return (end_dt - start_dt).total_seconds()
+
+ def _get_additional_metrics(self,
+ result: DataflowPipelineResult) -> dict[str, Any]:
+ job_id = result.job_id()
+ job = self.dataflow_client.get_job(job_id)
+ project = job.projectId
+ start_time, end_time = self._get_worker_time_interval(job_id)
+ if not start_time or not end_time:
+ logging.warning('Could not find valid worker start/end times.')
+ return {}
+
+ throughput_metrics = self._get_throughput_metrics(
+ project, job_id, start_time, end_time)
+ return {
+ **throughput_metrics,
+ "JobRuntimeSeconds": self._get_job_runtime(start_time, end_time),
+ }
diff --git a/website/www/site/content/en/performance/_index.md b/website/www/site/content/en/performance/_index.md
index f821b0f2508..45f819d23be 100644
--- a/website/www/site/content/en/performance/_index.md
+++ b/website/www/site/content/en/performance/_index.md
@@ -30,11 +30,22 @@ from a pipeline Job running on [Dataflow](/documentation/runners/dataflow/).
See the [glossary](/performance/glossary) for a list of the metrics and their
definition.
-# Measured Beam IOs
+# Measured Beam Java IOs
See the following pages for performance measures recorded when reading from and
writing to various Beam IOs.
- [BigQuery](/performance/bigquery)
- [BigTable](/performance/bigtable)
-- [TextIO](/performance/textio)
\ No newline at end of file
+- [TextIO](/performance/textio)
+
+# Measured Beam Python ML Pipelines
+
+See the following pages for performance measures recorded when running various Beam ML pipelines.
+
+- [PyTorch Language Modeling BERT base](/performance/pytorchbertbase)
+- [PyTorch Language Modeling BERT large](/performance/pytorchbertlarge)
+- [PyTorch Vision Classification Resnet 101](/performance/pytorchresnet101)
+- [PyTorch Vision Classification Resnet 152](/performance/pytorchresnet152)
+- [PyTorch Vision Classification Resnet 152 Tesla T4 GPU](/performance/pytorchresnet152tesla)
+- [TensorFlow MNIST Image Classification](/performance/tensorflowmnist)
\ No newline at end of file
diff --git a/website/www/site/content/en/performance/pytorchbertbase/_index.md b/website/www/site/content/en/performance/pytorchbertbase/_index.md
new file mode 100644
index 00000000000..9ef6cb455ca
--- /dev/null
+++ b/website/www/site/content/en/performance/pytorchbertbase/_index.md
@@ -0,0 +1,34 @@
+---
+title: "PyTorch Language Modeling BERT base Performance"
+---
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# PyTorch Language Modeling BERT base Performance
+
+The following graphs show various metrics when running Pytorch Language Modeling using Hugging Face bert-base-uncased model pipeline.
+See the [glossary](/performance/glossary) for definitions.
+
+## What is the estimated cost to run the pipeline?
+
+{{< performance_looks io="pytorchbertbase" read_or_write="write" section="cost" >}}
+
+## How has various metrics changed when running the pipeline for different Beam SDK versions?
+
+{{< performance_looks io="pytorchbertbase" read_or_write="write" section="version" >}}
+
+## How has various metrics changed over time when running the pipeline?
+
+{{< performance_looks io="pytorchbertbase" read_or_write="write" section="date" >}}
diff --git a/website/www/site/content/en/performance/pytorchbertlarge/_index.md b/website/www/site/content/en/performance/pytorchbertlarge/_index.md
new file mode 100644
index 00000000000..ea18712c101
--- /dev/null
+++ b/website/www/site/content/en/performance/pytorchbertlarge/_index.md
@@ -0,0 +1,34 @@
+---
+title: "PyTorch Language Modeling BERT large Performance"
+---
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# PyTorch Language Modeling BERT base Performance
+
+The following graphs show various metrics when running Pytorch Language Modeling using Hugging Face bert-large-uncased model pipeline.
+See the [glossary](/performance/glossary) for definitions.
+
+## What is the estimated cost to run the pipeline?
+
+{{< performance_looks io="pytorchbertlarge" read_or_write="write" section="cost" >}}
+
+## How has various metrics changed when running the pipeline for different Beam SDK versions?
+
+{{< performance_looks io="pytorchbertlarge" read_or_write="write" section="version" >}}
+
+## How has various metrics changed over time when running the pipeline?
+
+{{< performance_looks io="pytorchbertlarge" read_or_write="write" section="date" >}}
diff --git a/website/www/site/content/en/performance/pytorchresnet101/_index.md b/website/www/site/content/en/performance/pytorchresnet101/_index.md
new file mode 100644
index 00000000000..d65c5ec377f
--- /dev/null
+++ b/website/www/site/content/en/performance/pytorchresnet101/_index.md
@@ -0,0 +1,34 @@
+---
+title: "Pytorch Vision Classification with Resnet 101 Performance"
+---
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Pytorch Vision Classification with Resnet 101 Performance
+
+The following graphs show various metrics when running Pytorch Vision Classification with Resnet 101 pipeline.
+See the [glossary](/performance/glossary) for definitions.
+
+## What is the estimated cost to run the pipeline?
+
+{{< performance_looks io="pytorchresnet101" read_or_write="write" section="cost" >}}
+
+## How has various metrics changed when running the pipeline for different Beam SDK versions?
+
+{{< performance_looks io="pytorchresnet101" read_or_write="write" section="version" >}}
+
+## How has various metrics changed over time when running the pipeline?
+
+{{< performance_looks io="pytorchresnet101" read_or_write="write" section="date" >}}
diff --git a/website/www/site/content/en/performance/pytorchresnet152/_index.md b/website/www/site/content/en/performance/pytorchresnet152/_index.md
new file mode 100644
index 00000000000..1270eb1b4f3
--- /dev/null
+++ b/website/www/site/content/en/performance/pytorchresnet152/_index.md
@@ -0,0 +1,34 @@
+---
+title: "Pytorch Vision Classification with Resnet 152 Performance"
+---
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Pytorch Vision Classification with Resnet 152 Performance
+
+The following graphs show various metrics when running Pytorch Vision Classification with Resnet 152 pipeline.
+See the [glossary](/performance/glossary) for definitions.
+
+## What is the estimated cost to run the pipeline?
+
+{{< performance_looks io="pytorchresnet152" read_or_write="write" section="cost" >}}
+
+## How has various metrics changed when running the pipeline for different Beam SDK versions?
+
+{{< performance_looks io="pytorchresnet152" read_or_write="write" section="version" >}}
+
+## How has various metrics changed over time when running the pipeline?
+
+{{< performance_looks io="pytorchresnet152" read_or_write="write" section="date" >}}
diff --git a/website/www/site/content/en/performance/pytorchresnet152tesla/_index.md b/website/www/site/content/en/performance/pytorchresnet152tesla/_index.md
new file mode 100644
index 00000000000..cd03ce0d985
--- /dev/null
+++ b/website/www/site/content/en/performance/pytorchresnet152tesla/_index.md
@@ -0,0 +1,34 @@
+---
+title: "Pytorch Vision Classification with Resnet 152 with Tesla T4 GPU Performance"
+---
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Pytorch Vision Classification with Resnet 152 with Tesla T4 GPU Performance
+
+The following graphs show various metrics when running Pytorch Vision Classification with Resnet 152 with Tesla T4 GPU pipeline.
+See the [glossary](/performance/glossary) for definitions.
+
+## What is the estimated cost to run the pipeline?
+
+{{< performance_looks io="pytorchresnet152tesla" read_or_write="write" section="cost" >}}
+
+## How has various metrics changed when running the pipeline for different Beam SDK versions?
+
+{{< performance_looks io="pytorchresnet152tesla" read_or_write="write" section="version" >}}
+
+## How has various metrics changed over time when running the pipeline?
+
+{{< performance_looks io="pytorchresnet152tesla" read_or_write="write" section="date" >}}
diff --git a/website/www/site/content/en/performance/tensorflowmnist/_index.md b/website/www/site/content/en/performance/tensorflowmnist/_index.md
new file mode 100644
index 00000000000..350405fed1c
--- /dev/null
+++ b/website/www/site/content/en/performance/tensorflowmnist/_index.md
@@ -0,0 +1,34 @@
+---
+title: "TensorFlow MNIST Image Classification Performance"
+---
+
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# TensorFlow MNIST Image Classification Performance
+
+The following graphs show various metrics when running TensorFlow MNIST Image Classification pipeline.
+See the [glossary](/performance/glossary) for definitions.
+
+## What is the estimated cost to run the pipeline?
+
+{{< performance_looks io="tensorflowmnist" read_or_write="write" section="cost" >}}
+
+## How has various metrics changed when running the pipeline for different Beam SDK versions?
+
+{{< performance_looks io="tensorflowmnist" read_or_write="write" section="version" >}}
+
+## How has various metrics changed over time when running the pipeline?
+
+{{< performance_looks io="tensorflowmnist" read_or_write="write" section="date" >}}
diff --git a/website/www/site/data/performance.yaml b/website/www/site/data/performance.yaml
index dc375811c83..2c4ba241158 100644
--- a/website/www/site/data/performance.yaml
+++ b/website/www/site/data/performance.yaml
@@ -106,3 +106,99 @@ looks:
title: AvgInputThroughputBytesPerSec by Version
- id: fVVHhXCrHNgBG52TJsTjR8VbmWCCQnVN
title: AvgInputThroughputElementsPerSec by Version
+ pytorchbertbase:
+ write:
+ folder: 76
+ cost:
+ - id: Vybj7cBtbvVWJG63RRcYCTBC8TrD3Sdm
+ title: RunTime and EstimatedCost
+ date:
+ - id: DZfwm7T8kyVXzBkd7Hm65y8JNfNzZzYT
+ title: AvgThroughputBytesPerSec by Date
+ - id: ZDnG6kH55T2WPSD7yQh5cF6pkrQdRHKr
+ title: AvgThroughputElementsPerSec by Date
+ version:
+ - id: YCGWnm7S84qRcVm6kPKRwwgnKpg5xyJW
+ title: AvgThroughputBytesPerSec by Version
+ - id: 2dPXDTthFxDhvdypyHYNp7bSbMJggW6x
+ title: AvgThroughputElementsPerSec by Version
+ pytorchbertlarge:
+ write:
+ folder: 77
+ cost:
+ - id: gTN4qQbqFfJMWJKzwJHsXpjVV8McFbm8
+ title: RunTime and EstimatedCost
+ date:
+ - id: jGS2p6kTK9pZq94sYdqmNcz67PP6pKFd
+ title: AvgThroughputBytesPerSec by Date
+ - id: wfhCtgfnqM5YjRYbp4624fnyJcT2zXcT
+ title: AvgThroughputElementsPerSec by Date
+ version:
+ - id: Z3k29nwZrdCXJZdg5Yg7SSKDm2T4y8rZ
+ title: AvgThroughputBytesPerSec by Version
+ - id: D5g8qkqGKTpNqC8RV9cK2mPPD7rqJ8f4
+ title: AvgThroughputElementsPerSec by Version
+ pytorchresnet101:
+ write:
+ folder: 78
+ cost:
+ - id: DKbt3WmgTxnxXd5FKMtPvf5SgxYSByPT
+ title: RunTime and EstimatedCost
+ date:
+ - id: GDMn2mY45d4wpvw3tZpJhYnC6gpqysvn
+ title: AvgThroughputBytesPerSec by Date
+ - id: VnXf9SqntCd2SRw3Br2bgfkytVGdGxrV
+ title: AvgThroughputElementsPerSec by Date
+ version:
+ - id: cmWSXFn4Vp2pvpFJK3NNQg3mdTk7ywBC
+ title: AvgThroughputBytesPerSec by Version
+ - id: BpPdzhWWJttM8gcmQ4WSpFKX38BfHwbk
+ title: AvgThroughputElementsPerSec by Version
+ pytorchresnet152:
+ write:
+ folder: 79
+ cost:
+ - id: jkV2YJPv3MgqD22DRB65cbGNVjPDcJwT
+ title: RunTime and EstimatedCost
+ date:
+ - id: pvQwSM5JvxmJDcXpDJySctdYZkWDF69H
+ title: AvgThroughputBytesPerSec by Date
+ - id: JGctprgybxbfp2sBjspnBdRppmRXS5Sn
+ title: AvgThroughputElementsPerSec by Date
+ version:
+ - id: qc689x3JQxg5DWWVC4mBPqGCdx3hPSTG
+ title: AvgThroughputBytesPerSec by Version
+ - id: wS7Htr76CJ75gJ47tVP8ZT8rBw6BY3QW
+ title: AvgThroughputElementsPerSec by Version
+ pytorchresnet152tesla:
+ write:
+ folder: 80
+ cost:
+ - id: YD3mVwkS3976Cv7bCSSmDP5f4jXFsFRF
+ title: RunTime and EstimatedCost
+ date:
+ - id: 8r96B3vsfhTpwgz4FgH7xbH5KY8d5k4b
+ title: AvgThroughputBytesPerSec by Date
+ - id: whGvSJZzRbpvfYrqMhnsJRHWk3mKyF7r
+ title: AvgThroughputElementsPerSec by Date
+ version:
+ - id: hGVcdDzrSndZh68P9jrY5MMTCQ6wwrKb
+ title: AvgThroughputBytesPerSec by Version
+ - id: DVhGKTmJWknSvfQVPQ9FDrvPYgdJ2dFd
+ title: AvgThroughputElementsPerSec by Version
+ tensorflowmnist:
+ write:
+ folder: 75
+ cost:
+ - id: Vs9ZHMkCkrSgJF7FCPdQS5HwK8PQTyWb
+ title: RunTime and EstimatedCost
+ date:
+ - id: 7mYxWj4hDXQp2SZ28vMNTCZGhWcPQdwJ
+ title: AvgThroughputBytesPerSec by Date
+ - id: bWhWQ9t2jKGscc9ghgH77wRszTxwW8mM
+ title: AvgThroughputElementsPerSec by Date
+ version:
+ - id: y3jVqx2xKcZGpkMBTSCZCpGMPPFHrC8V
+ title: AvgThroughputBytesPerSec by Version
+ - id: YdD9SMWCDNJ7wCY4WZwyd2Jt9Ts38FY2
+ title: AvgThroughputElementsPerSec by Version