You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2023/07/21 19:02:43 UTC
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #315
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/315/display/redirect>
Changes:
------------------------------------------
[...truncated 475.18 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07211151-on0q-harness-7b5s,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07211151-on0q-harness-7d7n
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T18:59:00.793Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T18:59:00.955Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T18:59:00.980Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T19:01:56.644Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T19:01:56.779Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T19:01:56.849Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T19:01:59.084Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T19:01:59.154Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T19:01:59.211Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-21T19:02:00.127Z: JOB_MESSAGE_BASIC: Your project already contains 100 Dataflow-created metric descriptors, so new user metrics of the form custom.googleapis.com/* will not be created. However, all user metrics are also available in the metric dataflow.googleapis.com/job/user_counter. If you rely on the custom metrics, you can delete old / unused metric descriptors. See https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-21_11_51_34-17119747036653247977 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-21_11_51_34-17119747036653247977?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 29s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/kj73atbkkbpzq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal : beam_Inference_Python_Benchmarks_Dataflow #327
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/327/display/redirect>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #326
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/326/display/redirect>
Changes:
------------------------------------------
[...truncated 660.63 KB...]
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 663, in process_bundle
bundle_processor.process_bundle(instruction_id))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 1040, in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 232, in process_encoded
self.output(decoded_value)
File "apache_beam/runners/****/operations.py", line 568, in apache_beam.runners.****.operations.Operation.output
File "apache_beam/runners/****/operations.py", line 570, in apache_beam.runners.****.operations.Operation.output
File "apache_beam/runners/****/operations.py", line 261, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 1065, in apache_beam.runners.****.operations.SdfProcessSizedElements.process
File "apache_beam/runners/****/operations.py", line 1074, in apache_beam.runners.****.operations.SdfProcessSizedElements.process
File "apache_beam/runners/common.py", line 1454, in apache_beam.runners.common.DoFnRunner.process_with_sized_restriction
File "apache_beam/runners/common.py", line 819, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 983, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 839, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 985, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 916, in process
return self._run_inference(batch, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 885, in _run_inference
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 881, in _run_inference
result_generator = self._model_handler.run_inference(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 488, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 432, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 280, in run_inference
keys, self._unkeyed.run_inference(unkeyed_batch, model, inference_args))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 317, in run_inference
return self._inference_fn(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 150, in default_tensor_inference_fn
batched_tensors = _convert_to_device(batched_tensors, device)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 135, in _convert_to_device
examples = examples.to(device)
RuntimeError: CUDA error: misaligned address
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
[while running 'PyTorchRunInference/BeamML_RunInference-ptransform-73']
Worker ID: benchmark-tests-pytorch-i-07311243-i2rg-harness-zpsg
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T20:13:46.684Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T20:13:46.905Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T20:13:46.927Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T20:16:11.675Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 29 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T20:16:11.723Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T20:16:11.749Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-31_12_43_47-13608121215565686393 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-31_12_43_47-13608121215565686393?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py",> line 68, in <module>
PytorchVisionBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py",> line 58, in test
self.result = pytorch_image_classification.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_image_classification.py",> line 166, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 839, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 985, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 916, in process
return self._run_inference(batch, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 885, in _run_inference
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 881, in _run_inference
result_generator = self._model_handler.run_inference(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 488, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 432, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 280, in run_inference
keys, self._unkeyed.run_inference(unkeyed_batch, model, inference_args))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 317, in run_inference
return self._inference_fn(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 150, in default_tensor_inference_fn
batched_tensors = _convert_to_device(batched_tensors, device)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 135, in _convert_to_device
examples = examples.to(device)
RuntimeError: CUDA error: misaligned address
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 663, in process_bundle
bundle_processor.process_bundle(instruction_id))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 1040, in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 232, in process_encoded
self.output(decoded_value)
File "apache_beam/runners/****/operations.py", line 568, in apache_beam.runners.****.operations.Operation.output
File "apache_beam/runners/****/operations.py", line 570, in apache_beam.runners.****.operations.Operation.output
File "apache_beam/runners/****/operations.py", line 261, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 1065, in apache_beam.runners.****.operations.SdfProcessSizedElements.process
File "apache_beam/runners/****/operations.py", line 1074, in apache_beam.runners.****.operations.SdfProcessSizedElements.process
File "apache_beam/runners/common.py", line 1454, in apache_beam.runners.common.DoFnRunner.process_with_sized_restriction
File "apache_beam/runners/common.py", line 819, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 983, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 839, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 985, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 916, in process
return self._run_inference(batch, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 885, in _run_inference
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 881, in _run_inference
result_generator = self._model_handler.run_inference(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 488, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 432, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 280, in run_inference
keys, self._unkeyed.run_inference(unkeyed_batch, model, inference_args))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 317, in run_inference
return self._inference_fn(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 150, in default_tensor_inference_fn
batched_tensors = _convert_to_device(batched_tensors, device)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 135, in _convert_to_device
examples = examples.to(device)
RuntimeError: CUDA error: misaligned address
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
[while running 'PyTorchRunInference/BeamML_RunInference-ptransform-73']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 37m 10s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/buvrflpi4bnxu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #325
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/325/display/redirect?page=changes>
Changes:
[noreply] Bump github.com/fsouza/fake-gcs-server from 1.45.2 to 1.46.0 in /sdks
------------------------------------------
[...truncated 660.93 KB...]
File "apache_beam/runners/common.py", line 1454, in apache_beam.runners.common.DoFnRunner.process_with_sized_restriction
File "apache_beam/runners/common.py", line 819, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 983, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 839, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 985, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 916, in process
return self._run_inference(batch, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 885, in _run_inference
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 881, in _run_inference
result_generator = self._model_handler.run_inference(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 488, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 432, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 280, in run_inference
keys, self._unkeyed.run_inference(unkeyed_batch, model, inference_args))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 317, in run_inference
return self._inference_fn(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 150, in default_tensor_inference_fn
batched_tensors = _convert_to_device(batched_tensors, device)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 135, in _convert_to_device
examples = examples.to(device)
RuntimeError: CUDA error: misaligned address
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
[while running 'PyTorchRunInference/BeamML_RunInference-ptransform-73']
Worker ID: benchmark-tests-pytorch-i-07310755-xj4q-harness-r29j
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T15:25:06.253Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T15:25:06.417Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T15:25:06.441Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T15:27:30.039Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 28 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T15:27:30.072Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-31T15:27:30.092Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-31_07_55_33-4390270479772149698 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-31_07_55_33-4390270479772149698?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py",> line 68, in <module>
PytorchVisionBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py",> line 58, in test
self.result = pytorch_image_classification.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_image_classification.py",> line 166, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 839, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 985, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 916, in process
return self._run_inference(batch, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 885, in _run_inference
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 881, in _run_inference
result_generator = self._model_handler.run_inference(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 488, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 432, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 280, in run_inference
keys, self._unkeyed.run_inference(unkeyed_batch, model, inference_args))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 317, in run_inference
return self._inference_fn(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 150, in default_tensor_inference_fn
batched_tensors = _convert_to_device(batched_tensors, device)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 135, in _convert_to_device
examples = examples.to(device)
RuntimeError: CUDA error: misaligned address
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 663, in process_bundle
bundle_processor.process_bundle(instruction_id))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 1040, in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 232, in process_encoded
self.output(decoded_value)
File "apache_beam/runners/****/operations.py", line 568, in apache_beam.runners.****.operations.Operation.output
File "apache_beam/runners/****/operations.py", line 570, in apache_beam.runners.****.operations.Operation.output
File "apache_beam/runners/****/operations.py", line 261, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 1065, in apache_beam.runners.****.operations.SdfProcessSizedElements.process
File "apache_beam/runners/****/operations.py", line 1074, in apache_beam.runners.****.operations.SdfProcessSizedElements.process
File "apache_beam/runners/common.py", line 1454, in apache_beam.runners.common.DoFnRunner.process_with_sized_restriction
File "apache_beam/runners/common.py", line 819, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 983, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1513, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 625, in apache_beam.runners.common.SimpleInvoker.invoke_process
File "apache_beam/runners/common.py", line 1607, in apache_beam.runners.common._OutputHandler.handle_process_outputs
File "apache_beam/runners/common.py", line 1720, in apache_beam.runners.common._OutputHandler._write_value_to_tag
File "apache_beam/runners/****/operations.py", line 264, in apache_beam.runners.****.operations.SingletonElementConsumerSet.receive
File "apache_beam/runners/****/operations.py", line 951, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/****/operations.py", line 952, in apache_beam.runners.****.operations.DoOperation.process
File "apache_beam/runners/common.py", line 1425, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1423, in apache_beam.runners.common.DoFnRunner.process
File "apache_beam/runners/common.py", line 839, in apache_beam.runners.common.PerWindowInvoker.invoke_process
File "apache_beam/runners/common.py", line 985, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 916, in process
return self._run_inference(batch, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 885, in _run_inference
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 881, in _run_inference
result_generator = self._model_handler.run_inference(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 488, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 432, in run_inference
return self._base.run_inference(batch, model, inference_args)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 280, in run_inference
keys, self._unkeyed.run_inference(unkeyed_batch, model, inference_args))
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 317, in run_inference
return self._inference_fn(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 150, in default_tensor_inference_fn
batched_tensors = _convert_to_device(batched_tensors, device)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 135, in _convert_to_device
examples = examples.to(device)
RuntimeError: CUDA error: misaligned address
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
[while running 'PyTorchRunInference/BeamML_RunInference-ptransform-73']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 36m 47s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
Publishing failed.
The response from https://ge.apache.org/scans/publish/gradle/3.13.2/token was not from Gradle Enterprise.
The specified server address may be incorrect, or your network environment may be interfering.
Please report this problem to your Gradle Enterprise administrator via https://ge.apache.org/help and include the following via copy/paste:
----------
Gradle version: 7.6.2
Plugin version: 3.13.2
Request URL: https://ge.apache.org/scans/publish/gradle/3.13.2/token
Request ID: e6953b61-4ead-4ab9-8058-4a0397a802df
Response status code: 502
Response content type: text/html
----------
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #324
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/324/display/redirect>
Changes:
------------------------------------------
[...truncated 591.64 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07301209-pzcy-harness-wl92,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07301209-pzcy-harness-8hzh
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-30T19:18:48.466Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-30T19:18:48.632Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-30T19:18:48.655Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-30T19:21:09.148Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-30T19:21:09.206Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-30T19:21:09.284Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-30_12_09_06-2182982055131609381 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-30_12_09_06-2182982055131609381?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 16m 55s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/i5ylrvypshpwi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #323
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/323/display/redirect>
Changes:
------------------------------------------
[...truncated 591.63 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07291211-mwaa-harness-btkp,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07291211-mwaa-harness-z4vn
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-29T19:21:05.129Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-29T19:21:05.272Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-29T19:21:05.296Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-29T19:23:26.235Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-29T19:23:26.288Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-29T19:23:26.306Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-29_12_11_13-1612403527375256011 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-29_12_11_13-1612403527375256011?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 16m 58s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/g4ags5ig7qaky
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #322
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/322/display/redirect>
Changes:
------------------------------------------
[...truncated 176.44 KB...]
Using cached PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (736 kB)
Using cached regex-2023.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (772 kB)
Using cached requests-2.31.0-py3-none-any.whl (62 kB)
Using cached requests_mock-1.11.0-py2.py3-none-any.whl (28 kB)
Using cached scikit_learn-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.1 MB)
Using cached SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)
Using cached typing_extensions-4.7.1-py3-none-any.whl (33 kB)
Using cached botocore-1.31.14-py3-none-any.whl (11.1 MB)
Using cached certifi-2023.7.22-py3-none-any.whl (158 kB)
Using cached charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (199 kB)
Using cached dnspython-2.4.1-py3-none-any.whl (300 kB)
Using cached docker-6.1.3-py3-none-any.whl (148 kB)
Using cached exceptiongroup-1.1.2-py3-none-any.whl (14 kB)
Using cached execnet-2.0.2-py3-none-any.whl (37 kB)
Using cached google_cloud_resource_manager-1.10.2-py2.py3-none-any.whl (321 kB)
Using cached google_cloud_storage-2.10.0-py2.py3-none-any.whl (114 kB)
Using cached googleapis_common_protos-1.59.1-py2.py3-none-any.whl (224 kB)
Using cached greenlet-3.0.0a1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (613 kB)
Using cached grpcio_status-1.57.0rc1-py3-none-any.whl (5.1 kB)
Using cached msal-1.24.0b1-py2.py3-none-any.whl (90 kB)
Using cached pyparsing-3.1.0-py3-none-any.whl (102 kB)
Using cached threadpoolctl-3.2.0-py3-none-any.whl (15 kB)
Using cached urllib3-1.26.16-py2.py3-none-any.whl (143 kB)
Using cached PyMySQL-1.1.0-py3-none-any.whl (44 kB)
Using cached websocket_client-1.6.1-py3-none-any.whl (56 kB)
Using cached PyJWT-2.8.0-py3-none-any.whl (22 kB)
Building wheels for collected packages: apache-beam
Building wheel for apache-beam (setup.py): started
Building wheel for apache-beam (setup.py): finished with status 'done'
Created wheel for apache-beam: filename=apache_beam-2.50.0.dev0-py3-none-any.whl size=3252743 sha256=7251e8244b99c599ce02f2a4203c1049dc5fafddf37a977bfc44ae58f71bad58
Stored in directory: /home/jenkins/.cache/pip/wheels/d4/ad/b4/5dfba3e341b10ae80f4e32ab3ad25d368a3c6304b888eb7e37
Successfully built apache-beam
Installing collected packages: sortedcontainers, pytz, docopt, crcmod, zstandard, wrapt, websocket-client, urllib3, typing-extensions, threadpoolctl, tenacity, sqlparse, six, shapely, scipy, regex, pyyaml, pyparsing, pymysql, PyJWT, pyhamcrest, pycparser, pyasn1, pyarrow, psycopg2-binary, protobuf, portalocker, parameterized, overrides, orjson, objsize, mock, joblib, jmespath, iniconfig, idna, greenlet, google-crc32c, fasteners, fastavro, execnet, exceptiongroup, dnspython, dill, deprecation, cloudpickle, charset-normalizer, certifi, attrs, sqlalchemy, scikit-learn, rsa, requests, python-dateutil, pytest, pymongo, pydot, pyasn1-modules, proto-plus, isodate, hypothesis, httplib2, googleapis-common-protos, google-resumable-media, cffi, requests_mock, pytest-xdist, pytest-timeout, pandas, oauth2client, hdfs, grpcio-status, google-auth, freezegun, docker, cryptography, botocore, azure-core, testcontainers, s3transfer, grpc-google-iam-v1, google-auth-httplib2, google-apitools, google-api-core, azure-storage-blob, apache-beam, msal, google-cloud-core, boto3, msal-extensions, google-cloud-vision, google-cloud-videointelligence, google-cloud-storage, google-cloud-spanner, google-cloud-resource-manager, google-cloud-recommendations-ai, google-cloud-pubsub, google-cloud-language, google-cloud-dlp, google-cloud-datastore, google-cloud-bigtable, google-cloud-bigquery-storage, google-cloud-bigquery, google-cloud-pubsublite, google-cloud-aiplatform, azure-identity
Attempting uninstall: protobuf
Found existing installation: protobuf 4.24.0rc2
Uninstalling protobuf-4.24.0rc2:
Successfully uninstalled protobuf-4.24.0rc2
Successfully installed PyJWT-2.8.0 apache-beam-2.50.0.dev0 attrs-23.1.0 azure-core-1.28.0 azure-identity-1.14.0b2 azure-storage-blob-12.17.0 boto3-1.28.14 botocore-1.31.14 certifi-2023.7.22 cffi-1.15.1 charset-normalizer-3.2.0 cloudpickle-2.2.1 crcmod-1.7 cryptography-41.0.2 deprecation-2.1.0 dill-0.3.1.1 dnspython-2.4.1 docker-6.1.3 docopt-0.6.2 exceptiongroup-1.1.2 execnet-2.0.2 fastavro-1.8.2 fasteners-0.18 freezegun-1.2.2 google-api-core-2.12.0.dev0 google-apitools-0.5.31 google-auth-2.22.0 google-auth-httplib2-0.1.0 google-cloud-aiplatform-1.28.1 google-cloud-bigquery-3.11.4 google-cloud-bigquery-storage-2.22.0 google-cloud-bigtable-2.20.0 google-cloud-core-2.3.3 google-cloud-datastore-2.16.1 google-cloud-dlp-3.12.2 google-cloud-language-2.10.1 google-cloud-pubsub-2.18.1 google-cloud-pubsublite-1.8.3 google-cloud-recommendations-ai-0.10.4 google-cloud-resource-manager-1.10.2 google-cloud-spanner-3.38.0 google-cloud-storage-2.10.0 google-cloud-videointelligence-2.11.3 google-cloud-vision-3.4.4 google-crc32c-1.5.0 google-resumable-media-2.5.0 googleapis-common-protos-1.59.1 greenlet-3.0.0a1 grpc-google-iam-v1-0.12.6 grpcio-status-1.57.0rc1 hdfs-2.7.0 httplib2-0.22.0 hypothesis-6.82.0 idna-3.4 iniconfig-2.0.0 isodate-0.6.1 jmespath-1.0.1 joblib-1.3.1 mock-5.1.0 msal-1.24.0b1 msal-extensions-1.0.0 oauth2client-4.1.3 objsize-0.6.1 orjson-3.9.2 overrides-6.5.0 pandas-1.5.3 parameterized-0.9.0 portalocker-2.7.0 proto-plus-1.22.3 protobuf-4.23.4 psycopg2-binary-2.9.6 pyarrow-11.0.0 pyasn1-0.5.0 pyasn1-modules-0.3.0 pycparser-2.21 pydot-1.4.2 pyhamcrest-2.0.4 pymongo-4.4.1 pymysql-1.1.0 pyparsing-3.1.0 pytest-7.4.0 pytest-timeout-2.1.0 pytest-xdist-3.3.1 python-dateutil-2.8.2 pytz-2023.3 pyyaml-6.0.1 regex-2023.6.3 requests-2.31.0 requests_mock-1.11.0 rsa-4.9 s3transfer-0.6.1 scikit-learn-1.3.0 scipy-1.10.1 shapely-1.8.5.post1 six-1.16.0 sortedcontainers-2.4.0 sqlalchemy-1.4.49 sqlparse-0.4.4 tenacity-8.2.2 testcontainers-3.7.1 threadpoolctl-3.2.0 typing-extensions-4.7.1 urllib3-1.26.16 websocket-client-1.6.1 wrapt-1.15.0 zstandard-0.21.0
> Task :sdks:python:apache_beam:testing:load_tests:run
Collecting torch>=1.7.1 (from -r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl (619.9 MB)
Collecting torchvision>=0.8.2 (from -r apache_beam/ml/inference/torch_tests_requirements.txt (line 19))
Using cached torchvision-0.15.2-cp38-cp38-manylinux1_x86_64.whl (33.8 MB)
Collecting pillow>=8.0.0 (from -r apache_beam/ml/inference/torch_tests_requirements.txt (line 20))
Obtaining dependency information for pillow>=8.0.0 from https://files.pythonhosted.org/packages/ff/8c/5927a58c43ebc16e508eef325fdc6473b569e2474d3b4be49798aa371007/Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl.metadata
Using cached Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl.metadata (9.5 kB)
Collecting transformers>=4.18.0 (from -r apache_beam/ml/inference/torch_tests_requirements.txt (line 21))
Obtaining dependency information for transformers>=4.18.0 from https://files.pythonhosted.org/packages/21/02/ae8e595f45b6c8edee07913892b3b41f5f5f273962ad98851dc6a564bbb9/transformers-4.31.0-py3-none-any.whl.metadata
Using cached transformers-4.31.0-py3-none-any.whl.metadata (116 kB)
Requirement already satisfied: filelock in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18)) (3.12.2)
Requirement already satisfied: typing-extensions in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18)) (4.7.1)
Collecting sympy (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached sympy-1.12-py3-none-any.whl (5.7 MB)
Collecting networkx (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached networkx-3.1-py3-none-any.whl (2.1 MB)
Collecting jinja2 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached Jinja2-3.1.2-py3-none-any.whl (133 kB)
Collecting nvidia-cuda-nvrtc-cu11==11.7.99 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)
Collecting nvidia-cuda-runtime-cu11==11.7.99 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)
Collecting nvidia-cuda-cupti-cu11==11.7.101 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl (11.8 MB)
Collecting nvidia-cudnn-cu11==8.5.0.96 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)
Collecting nvidia-cublas-cu11==11.10.3.66 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)
Collecting nvidia-cufft-cu11==10.9.0.58 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl (168.4 MB)
Collecting nvidia-curand-cu11==10.2.10.91 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl (54.6 MB)
Collecting nvidia-cusolver-cu11==11.4.0.1 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl (102.6 MB)
Collecting nvidia-cusparse-cu11==11.7.4.91 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl (173.2 MB)
Collecting nvidia-nccl-cu11==2.14.3 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl (177.1 MB)
Collecting nvidia-nvtx-cu11==11.7.91 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl (98 kB)
Collecting triton==2.0.0 (from torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.2 MB)
Requirement already satisfied: setuptools in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from nvidia-cublas-cu11==11.10.3.66->torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18)) (68.0.0)
Requirement already satisfied: wheel in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from nvidia-cublas-cu11==11.10.3.66->torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18)) (0.41.0)
Collecting cmake (from triton==2.0.0->torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Obtaining dependency information for cmake from https://files.pythonhosted.org/packages/14/b8/06f8fdc4687af3d3d8d95461d97737df2f144acd28eff65a3c47c29d0152/cmake-3.27.0-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata
Using cached cmake-3.27.0-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (6.7 kB)
Collecting lit (from triton==2.0.0->torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached lit-16.0.6-py3-none-any.whl
Requirement already satisfied: numpy in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from torchvision>=0.8.2->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 19)) (1.24.4)
Requirement already satisfied: requests in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from torchvision>=0.8.2->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 19)) (2.31.0)
Collecting huggingface-hub<1.0,>=0.14.1 (from transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21))
Obtaining dependency information for huggingface-hub<1.0,>=0.14.1 from https://files.pythonhosted.org/packages/7f/c4/adcbe9a696c135578cabcbdd7331332daad4d49b7c43688bc2d36b3a47d2/huggingface_hub-0.16.4-py3-none-any.whl.metadata
Using cached huggingface_hub-0.16.4-py3-none-any.whl.metadata (12 kB)
Requirement already satisfied: packaging>=20.0 in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21)) (23.1)
Requirement already satisfied: pyyaml>=5.1 in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21)) (6.0.1)
Requirement already satisfied: regex!=2019.12.17 in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21)) (2023.6.3)
Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21))
Using cached tokenizers-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)
Collecting safetensors>=0.3.1 (from transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21))
Using cached safetensors-0.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)
Collecting tqdm>=4.27 (from transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21))
Using cached tqdm-4.65.0-py3-none-any.whl (77 kB)
Collecting fsspec (from huggingface-hub<1.0,>=0.14.1->transformers>=4.18.0->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 21))
Obtaining dependency information for fsspec from https://files.pythonhosted.org/packages/e3/bd/4c0a4619494188a9db5d77e2100ab7d544a42e76b2447869d8e124e981d8/fsspec-2023.6.0-py3-none-any.whl.metadata
Using cached fsspec-2023.6.0-py3-none-any.whl.metadata (6.7 kB)
Collecting MarkupSafe>=2.0 (from jinja2->torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Obtaining dependency information for MarkupSafe>=2.0 from https://files.pythonhosted.org/packages/de/e2/32c14301bb023986dff527a49325b6259cab4ebb4633f69de54af312fc45/MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata
Using cached MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.0 kB)
Requirement already satisfied: charset-normalizer<4,>=2 in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from requests->torchvision>=0.8.2->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 19)) (3.2.0)
Requirement already satisfied: idna<4,>=2.5 in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from requests->torchvision>=0.8.2->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 19)) (3.4)
Requirement already satisfied: urllib3<3,>=1.21.1 in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from requests->torchvision>=0.8.2->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 19)) (1.26.16)
Requirement already satisfied: certifi>=2017.4.17 in <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/lib/python3.8/site-packages> (from requests->torchvision>=0.8.2->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 19)) (2023.7.22)
Collecting mpmath>=0.19 (from sympy->torch>=1.7.1->-r apache_beam/ml/inference/torch_tests_requirements.txt (line 18))
Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)
Using cached Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl (3.4 MB)
Using cached transformers-4.31.0-py3-none-any.whl (7.4 MB)
Using cached huggingface_hub-0.16.4-py3-none-any.whl (268 kB)
Using cached MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)
Using cached cmake-3.27.0-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (26.0 MB)
Using cached fsspec-2023.6.0-py3-none-any.whl (163 kB)
Installing collected packages: tokenizers, safetensors, mpmath, lit, cmake, tqdm, sympy, pillow, nvidia-nvtx-cu11, nvidia-nccl-cu11, nvidia-cusparse-cu11, nvidia-curand-cu11, nvidia-cufft-cu11, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cuda-cupti-cu11, nvidia-cublas-cu11, networkx, MarkupSafe, fsspec, nvidia-cusolver-cu11, nvidia-cudnn-cu11, jinja2, huggingface-hub, transformers, triton, torch, torchvision
Successfully installed MarkupSafe-2.1.3 cmake-3.27.0 fsspec-2023.6.0 huggingface-hub-0.16.4 jinja2-3.1.2 lit-16.0.6 mpmath-1.3.0 networkx-3.1 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-cupti-cu11-11.7.101 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 nvidia-cufft-cu11-10.9.0.58 nvidia-curand-cu11-10.2.10.91 nvidia-cusolver-cu11-11.4.0.1 nvidia-cusparse-cu11-11.7.4.91 nvidia-nccl-cu11-2.14.3 nvidia-nvtx-cu11-11.7.91 pillow-10.0.0 safetensors-0.3.1 sympy-1.12 tokenizers-0.13.3 torch-2.0.1 torchvision-0.15.2 tqdm-4.65.0 transformers-4.31.0 triton-2.0.0
INFO:root:Device is set to CPU
INFO:apache_beam.internal.gcp.auth:Setting socket default timeout to 60 seconds.
INFO:apache_beam.internal.gcp.auth:socket default timeout is 60.0 seconds.
INFO:apache_beam.runners.portability.stager:Executing command: ['<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/build/gradleenv/1329484227/bin/python',> '-m', 'pip', 'download', '--dest', '/tmp/dataflow-requirements-cache', '-r', '/tmp/tmpfqhij9gu/tmp_requirements.txt', '--exists-action', 'i', '--no-deps', '--implementation', 'cp', '--abi', 'cp38', '--platform', 'manylinux2014_x86_64']
INFO:apache_beam.runners.portability.stager:Copying Beam SDK "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/build/apache-beam.tar.gz"> to staging location.
INFO:apache_beam.runners.dataflow.dataflow_runner:Pipeline has additional dependencies to be installed in SDK **** container, consider using the SDK container image pre-building workflow to avoid repetitive installations. Learn more on https://cloud.google.com/dataflow/docs/guides/using-custom-containers#prebuild
INFO:root:Using provided Python SDK container image: gcr.io/cloud-dataflow/v1beta3/beam_python3.8_sdk:beam-master-20230717
INFO:root:Python SDK container image set to "gcr.io/cloud-dataflow/v1beta3/beam_python3.8_sdk:beam-master-20230717" for Docker environment
INFO:apache_beam.runners.portability.fn_api_runner.translations:==================== <function pack_combiners at 0x7f3e56878d30> ====================
INFO:apache_beam.runners.portability.fn_api_runner.translations:==================== <function sort_stages at 0x7f3e56879550> ====================
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/requirements.txt...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/requirements.txt in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit-learn-1.0.2.tar.gz...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit-learn-1.0.2.tar.gz in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/mock-2.0.0-py2.py3-none-any.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/mock-2.0.0-py2.py3-none-any.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/seaborn-0.12.2-py3-none-any.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/seaborn-0.12.2-py3-none-any.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/PyHamcrest-1.10.1-py3-none-any.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/PyHamcrest-1.10.1-py3-none-any.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/transformers-4.31.0-py3-none-any.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/transformers-4.31.0-py3-none-any.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/inflection-0.5.1-py2.py3-none-any.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/inflection-0.5.1-py2.py3-none-any.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/beautifulsoup4-4.12.2-py3-none-any.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/beautifulsoup4-4.12.2-py3-none-any.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/parameterized-0.7.5-py2.py3-none-any.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/parameterized-0.7.5-py2.py3-none-any.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl in 41 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/torchvision-0.15.2-cp38-cp38-manylinux1_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/torchvision-0.15.2-cp38-cp38-manylinux1_x86_64.whl in 2 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/Pillow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/Pillow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 1 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/matplotlib-3.7.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 1 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 1 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 1 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 1 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/scikit_learn-1.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/dataflow_python_sdk.tar...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/dataflow_python_sdk.tar in 0 seconds.
INFO:apache_beam.runners.dataflow.internal.apiclient:Starting GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/pipeline.pb...
INFO:apache_beam.runners.dataflow.internal.apiclient:Completed GCS upload to gs://temp-storage-for-perf-tests/loadtests/benchmark-tests-pytorch-imagenet-python0728161153.1690571366.053757/pipeline.pb in 0 seconds.
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py",> line 68, in <module>
PytorchVisionBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_image_classification_benchmarks.py",> line 58, in test
self.result = pytorch_image_classification.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_image_classification.py",> line 165, in run
result = pipeline.run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/test_pipeline.py",> line 112, in run
result = super().run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/pipeline.py",> line 550, in run
return Pipeline.from_runner_api(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/pipeline.py",> line 577, in run
return self.runner.run_pipeline(self, self._options)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 472, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/utils/retry.py",> line 275, in wrapper
return fun(*args, **kwargs)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",> line 720, in create_job
return self.submit_job_description(job)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/utils/retry.py",> line 275, in wrapper
return fun(*args, **kwargs)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",> line 838, in submit_job_description
raise DataflowJobAlreadyExistsError(
apache_beam.runners.dataflow.internal.apiclient.DataflowJobAlreadyExistsError: There is already active job named benchmark-tests-pytorch-imagenet-python0728161153 with id: 2023-07-28_12_09_44-4725819878998581239. If you want to submit a second job, try again by setting a different name using --job_name.
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 37s
15 actionable tasks: 5 executed, 10 up-to-date
Publishing build scan...
https://ge.apache.org/s/p6jglafziy6za
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #321
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/321/display/redirect>
Changes:
------------------------------------------
[...truncated 489.04 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07271152-e7qg-harness-1dmt,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07271152-e7qg-harness-0gx3
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-27T19:00:16.500Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-27T19:00:16.755Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-27T19:00:16.818Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-27T19:02:59.545Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-27T19:02:59.608Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-27T19:02:59.634Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-27_11_52_46-605136603281747057 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-27_11_52_46-605136603281747057?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 33s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/6h7jmjneuj53o
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #320
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/320/display/redirect>
Changes:
------------------------------------------
[...truncated 470.06 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07261151-vogd-harness-3dh4,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07261151-vogd-harness-vvxg
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-26T18:58:39.339Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-26T18:58:39.495Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-26T18:58:39.519Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-26T19:01:08.236Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-26T19:01:08.275Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-26T19:01:08.306Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-26_11_51_07-7589597009195410199 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-26_11_51_07-7589597009195410199?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.6.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 29s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/7gkp6a4q2fkfc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #319
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/319/display/redirect>
Changes:
------------------------------------------
[...truncated 456.80 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07251153-cjm0-harness-v44p,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07251153-cjm0-harness-bzf8
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-25T19:01:23.359Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-25T19:01:23.519Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-25T19:01:23.537Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-25T19:04:02.891Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-25T19:04:02.924Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-25T19:04:02.946Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-25_11_53_50-7646267992334101160 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-25_11_53_50-7646267992334101160?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 33s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/iib4qri5vha6o
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #318
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/318/display/redirect?page=changes>
Changes:
[noreply] Fix multi_process_shared.py release with AutoProxy (#27607)
------------------------------------------
[...truncated 451.02 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07241151-ovii-harness-jp82,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07241151-ovii-harness-k9hc
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-24T18:58:59.973Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-24T18:59:00.143Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-24T18:59:00.159Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-24T19:01:25.159Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-24T19:01:25.261Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-24T19:01:25.322Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-24_11_51_26-3600027418746932601 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-24_11_51_26-3600027418746932601?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 3s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/jriplqm2ba2j4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #317
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/317/display/redirect>
Changes:
------------------------------------------
[...truncated 478.97 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07231147-ty2y-harness-808g,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07231147-ty2y-harness-b7l1
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-23T18:55:27.726Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-23T18:55:27.917Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-23T18:55:27.935Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-23T18:57:49.516Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-23T18:57:49.549Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-23T18:57:49.573Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-23_11_47_56-3855905881840448368 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-23_11_47_56-3855905881840448368?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 59s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/5ppfngy7gluz6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_Inference_Python_Benchmarks_Dataflow #316
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/316/display/redirect>
Changes:
------------------------------------------
[...truncated 467.27 KB...]
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07221147-972z-harness-vthc,
Root cause: Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
Worker ID: benchmark-tests-pytorch-l-07221147-972z-harness-qqv7
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-22T18:54:42.204Z: JOB_MESSAGE_DETAILED: Cleaning up.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-22T18:54:42.401Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-22T18:54:42.418Z: JOB_MESSAGE_BASIC: Stopping **** pool...
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-22T18:57:03.959Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 250 to 0.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-22T18:57:03.996Z: JOB_MESSAGE_BASIC: Worker pool stopped.
INFO:apache_beam.runners.dataflow.dataflow_runner:2023-07-22T18:57:04.021Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
INFO:apache_beam.runners.dataflow.dataflow_runner:Job 2023-07-22_11_47_29-3672668500935564196 is in state JOB_STATE_FAILED
ERROR:apache_beam.runners.dataflow.dataflow_runner:Console URL: https://console.cloud.google.com/dataflow/jobs/<RegionId>/2023-07-22_11_47_29-3672668500935564196?project=<ProjectId>
Traceback (most recent call last):
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 42, in <module>
PytorchLanguageModelingBenchmarkTest().run()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/load_test.py",> line 148, in run
self.test()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/benchmarks/inference/pytorch_language_modeling_benchmarks.py",> line 35, in test
self.result = pytorch_language_modeling.run(
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/examples/inference/pytorch_language_modeling.py",> line 210, in run
result.wait_until_finish()
File "<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 756, in wait_until_finish
raise DataflowRuntimeException(
apache_beam.runners.dataflow.dataflow_runner.DataflowRuntimeException: Dataflow pipeline failed. State: FAILED, Error:
Traceback (most recent call last):
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids".
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 297, in _execute
response = task()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 372, in <lambda>
lambda: self.create_****().do_instruction(request), request)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 625, in do_instruction
return getattr(self, request_type)(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 656, in process_bundle
bundle_processor = self.bundle_processor_cache.get(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/sdk_****.py", line 487, in get
processor = bundle_processor.BundleProcessor(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/runners/****/bundle_processor.py", line 903, in __init__
op.setup(self.data_sampler)
File "apache_beam/runners/****/operations.py", line 875, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/****/operations.py", line 925, in apache_beam.runners.****.operations.DoOperation.setup
File "apache_beam/runners/common.py", line 1493, in apache_beam.runners.common.DoFnRunner.setup
File "apache_beam/runners/common.py", line 1489, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 1533, in apache_beam.runners.common.DoFnRunner._reraise_augmented
File "apache_beam/runners/common.py", line 1487, in apache_beam.runners.common.DoFnRunner._invoke_lifecycle_method
File "apache_beam/runners/common.py", line 553, in apache_beam.runners.common.DoFnInvoker.invoke_setup
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 873, in setup
self._model = self._load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 849, in _load_model
model = self._shared_model_handle.acquire(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 305, in acquire
return _shared_map.acquire(self._key, constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 246, in acquire
result = control_block.acquire(constructor_fn, tag)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/utils/shared.py", line 139, in acquire
result = constructor_fn()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 834, in load
model = self._model_handler.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/base.py", line 270, in load_model
return self._unkeyed.load_model()
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 501, in load_model
model, device = _load_model(
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 119, in _load_model
raise e
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/apache_beam/ml/inference/pytorch_inference.py", line 102, in _load_model
model.load_state_dict(state_dict)
File "/opt/apache/beam-venv/beam-venv-****-sdk-0-0/lib/python3.8/site-packages/torch/nn/modules/module.py", line 2041, in load_state_dict
raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for BertForMaskedLM:
Unexpected key(s) in state_dict: "bert.embeddings.position_ids". [while running 'PyTorchRunInference/BeamML_RunInference-ptransform-81']
> Task :sdks:python:apache_beam:testing:load_tests:run FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://ci-beam.apache.org/job/beam_Inference_Python_Benchmarks_Dataflow/ws/src/sdks/python/apache_beam/testing/load_tests/build.gradle'> line: 63
* What went wrong:
Execution failed for task ':sdks:python:apache_beam:testing:load_tests:run'.
> error occurred
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.5.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 34s
15 actionable tasks: 4 executed, 11 up-to-date
Publishing build scan...
https://ge.apache.org/s/uomowm6yziwds
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org