You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2024/01/02 12:32:41 UTC
Build failed in Jenkins: beam_PostCommit_Python_Examples_Direct #2750
See <https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/2750/display/redirect>
Changes:
------------------------------------------
[...truncated 796.27 KB...]
return getattr(self, request_type)(
apache_beam/runners/worker/sdk_worker.py:677: in process_bundle
bundle_processor.process_bundle(instruction_id))
apache_beam/runners/worker/bundle_processor.py:1113: in process_bundle
input_op_by_transform_id[element.transform_id].process_encoded(
apache_beam/runners/worker/bundle_processor.py:237: in process_encoded
self.output(decoded_value)
apache_beam/runners/worker/operations.py:572: in output
_cast_to_receiver(self.receivers[output_index]).receive(windowed_value)
apache_beam/runners/worker/operations.py:266: in receive
self.consumer.process(windowed_value)
apache_beam/runners/worker/operations.py:954: in process
delayed_applications = self.dofn_runner.process(o)
apache_beam/runners/common.py:1437: in process
self._reraise_augmented(exn, windowed_value)
apache_beam/runners/common.py:1547: in _reraise_augmented
raise new_exn
apache_beam/runners/common.py:1435: in process
return self.do_fn_invoker.invoke_process(windowed_value)
apache_beam/runners/common.py:851: in invoke_process
self._invoke_process_per_window(
apache_beam/runners/common.py:995: in _invoke_process_per_window
self.output_handler.handle_process_outputs(
apache_beam/runners/common.py:1611: in handle_process_outputs
for result in results:
apache_beam/io/iobase.py:1248: in <genexpr>
return (
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <apache_beam.io.textio._TextSink object at 0x7f628bcc96a0>
init_result = 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/9ebea183-177f-464c-b256-9530837ccb86/beam-temp-results-b3ccd4eca96a11eeaacdb9054ff9f18d'
writer_results = ['gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/9ebea183-177f-464c-b256-9530837ccb86/beam-temp-results-b3ccd4eca96a11eeaacdb9054ff9f18d/8ab764d7-5ec6-4ce1-a40e-dc225f0332c3.results']
unused_pre_finalize_results = <apache_beam.pvalue.EmptySideInput object at 0x7f628bcbcbb0>
@check_accessible(['file_path_prefix'])
def finalize_write(
self, init_result, writer_results, unused_pre_finalize_results):
writer_results = sorted(writer_results)
num_shards = len(writer_results)
src_files, dst_files, delete_files, num_skipped = (
self._check_state_for_finalize_write(writer_results, num_shards))
num_skipped += len(delete_files)
FileSystems.delete(delete_files)
num_shards_to_finalize = len(src_files)
min_threads = min(num_shards_to_finalize, FileBasedSink._MAX_RENAME_THREADS)
num_threads = max(1, min_threads)
chunk_size = FileSystems.get_chunk_size(self.file_path_prefix.get())
source_file_batch = [
src_files[i:i + chunk_size]
for i in range(0, len(src_files), chunk_size)
]
destination_file_batch = [
dst_files[i:i + chunk_size]
for i in range(0, len(dst_files), chunk_size)
]
if num_shards_to_finalize:
_LOGGER.info(
'Starting finalize_write threads with num_shards: %d (skipped: %d), '
'batches: %d, num_threads: %d',
num_shards_to_finalize,
num_skipped,
len(source_file_batch),
num_threads)
start_time = time.time()
# Use a thread pool for renaming operations.
def _rename_batch(batch):
"""_rename_batch executes batch rename operations."""
source_files, destination_files = batch
exceptions = []
try:
FileSystems.rename(source_files, destination_files)
return exceptions
except BeamIOError as exp:
if exp.exception_details is None:
raise
for (src, dst), exception in exp.exception_details.items():
if exception:
_LOGGER.error(
('Exception in _rename_batch. src: %s, '
'dst: %s, err: %s'),
src,
dst,
exception)
exceptions.append(exception)
else:
_LOGGER.debug('Rename successful: %s -> %s', src, dst)
return exceptions
exception_batches = util.run_using_threadpool(
_rename_batch,
list(zip(source_file_batch, destination_file_batch)),
num_threads)
all_exceptions = [
e for exception_batch in exception_batches for e in exception_batch
]
if all_exceptions:
> raise Exception(
'Encountered exceptions in finalize_write: %s' % all_exceptions)
E Exception: Encountered exceptions in finalize_write: [503] [while running 'WriteToText/Write/WriteImpl/FinalizeWrite']
apache_beam/io/filebasedsink.py:352: Exception
------------------------------ Captured log call -------------------------------
WARNING apache_beam.options.pipeline_options:pipeline_options.py:372 Discarding unparseable args: ['--sleep_secs=20', '--kms_key_name=projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test']
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function annotate_downstream_side_inputs at 0x7f62d0f0b1f0> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function fix_side_input_pcoll_coders at 0x7f62d0f0b310> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function pack_combiners at 0x7f62d0f0b820> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function lift_combiners at 0x7f62d0f0b8b0> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function expand_sdf at 0x7f62d0f0ba60> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function expand_gbk at 0x7f62d0f0baf0> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function sink_flattens at 0x7f62d0f0bc10> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function greedily_fuse at 0x7f62d0f0bca0> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function read_to_impulse at 0x7f62d0f0bd30> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function impulse_to_input at 0x7f62d0f0bdc0> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function sort_stages at 0x7f62d0ef5040> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function add_impulse_to_dangling_transforms at 0x7f62d0ef5160> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function setup_timer_mapping at 0x7f62d0f0bf70> ====================
INFO apache_beam.runners.portability.fn_api_runner.translations:translations.py:712 ==================== <function populate_data_channel_coders at 0x7f62d0ef50d0> ====================
INFO apache_beam.runners.worker.statecache:statecache.py:234 Creating state cache with size 104857600
INFO apache_beam.runners.portability.fn_api_runner.worker_handlers:worker_handlers.py:922 Created Worker handler <apache_beam.runners.portability.fn_api_runner.worker_handlers.EmbeddedWorkerHandler object at 0x7f62989ae130> for environment ref_Environment_default_environment_1 (beam:env:embedded_python:v1, b'')
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:579 Started BigQuery job: <JobReference
location: 'US'
projectId: 'apache-beam-testing'>
bq show -j --format=prettyjson --project_id=apache-beam-testing None
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:463 Using location 'US' from table <TableReference
datasetId: 'samples'
projectId: 'publicdata'
tableId: 'shakespeare'> referenced by query select UNIQUE(word) from publicdata:samples.shakespeare
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:816 Dataset apache-beam-testing:beam_temp_dataset_3531b7bda85a4c40a2bfd4d6f53b96d5 does not exist so we will create it as temporary with location=US
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:579 Started BigQuery job: <JobReference
jobId: 'beam_bq_job_QUERY_BQ_EXPORT_JOB_12af46d8-2_1704198625_490'
location: 'US'
projectId: 'apache-beam-testing'>
bq show -j --format=prettyjson --project_id=apache-beam-testing beam_bq_job_QUERY_BQ_EXPORT_JOB_12af46d8-2_1704198625_490
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_QUERY_BQ_EXPORT_JOB_12af46d8-2_1704198625_490 status: RUNNING
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_QUERY_BQ_EXPORT_JOB_12af46d8-2_1704198625_490 status: DONE
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:579 Started BigQuery job: <JobReference
jobId: 'beam_bq_job_EXPORT_BQ_EXPORT_JOB_12af46d8-2_1704198631_202'
location: 'US'
projectId: 'apache-beam-testing'>
bq show -j --format=prettyjson --project_id=apache-beam-testing beam_bq_job_EXPORT_BQ_EXPORT_JOB_12af46d8-2_1704198631_202
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_EXPORT_BQ_EXPORT_JOB_12af46d8-2_1704198631_202 status: RUNNING
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_EXPORT_BQ_EXPORT_JOB_12af46d8-2_1704198631_202 status: DONE
INFO apache_beam.io.gcp.gcsio:gcsio.py:491 Finished listing 1 files in 0.0649564266204834 seconds.
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:579 Started BigQuery job: <JobReference
location: 'US'
projectId: 'apache-beam-testing'>
bq show -j --format=prettyjson --project_id=apache-beam-testing None
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:463 Using location 'US' from table <TableReference
datasetId: 'samples'
projectId: 'publicdata'
tableId: 'shakespeare'> referenced by query select UNIQUE(corpus) from publicdata:samples.shakespeare
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:816 Dataset apache-beam-testing:beam_temp_dataset_b8ca7bbf3b01440080ca4fd94ecb2481 does not exist so we will create it as temporary with location=US
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:579 Started BigQuery job: <JobReference
jobId: 'beam_bq_job_QUERY_BQ_EXPORT_JOB_49d87921-0_1704198638_560'
location: 'US'
projectId: 'apache-beam-testing'>
bq show -j --format=prettyjson --project_id=apache-beam-testing beam_bq_job_QUERY_BQ_EXPORT_JOB_49d87921-0_1704198638_560
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_QUERY_BQ_EXPORT_JOB_49d87921-0_1704198638_560 status: RUNNING
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_QUERY_BQ_EXPORT_JOB_49d87921-0_1704198638_560 status: DONE
INFO apache_beam.io.gcp.bigquery_tools:bigquery_tools.py:579 Started BigQuery job: <JobReference
jobId: 'beam_bq_job_EXPORT_BQ_EXPORT_JOB_49d87921-0_1704198644_273'
location: 'US'
projectId: 'apache-beam-testing'>
bq show -j --format=prettyjson --project_id=apache-beam-testing beam_bq_job_EXPORT_BQ_EXPORT_JOB_49d87921-0_1704198644_273
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_EXPORT_BQ_EXPORT_JOB_49d87921-0_1704198644_273 status: RUNNING
INFO root:bigquery_tools.py:652 Job apache-beam-testing:US.beam_bq_job_EXPORT_BQ_EXPORT_JOB_49d87921-0_1704198644_273 status: DONE
INFO apache_beam.io.gcp.gcsio:gcsio.py:491 Finished listing 1 files in 0.058737754821777344 seconds.
INFO apache_beam.io.gcp.gcsio:gcsio.py:491 Finished listing 1 files in 0.057088613510131836 seconds.
INFO apache_beam.io.gcp.gcsio:gcsio.py:491 Finished listing 1 files in 0.06122994422912598 seconds.
INFO apache_beam.io.gcp.gcsio:gcsio.py:491 Finished listing 1 files in 0.05639934539794922 seconds.
INFO apache_beam.io.filebasedsink:filebasedsink.py:310 Starting finalize_write threads with num_shards: 1 (skipped: 0), batches: 1, num_threads: 1
ERROR apache_beam.io.filebasedsink:filebasedsink.py:332 Exception in _rename_batch. src: gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/9ebea183-177f-464c-b256-9530837ccb86/beam-temp-results-b3ccd4eca96a11eeaacdb9054ff9f18d/8ab764d7-5ec6-4ce1-a40e-dc225f0332c3.results, dst: gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/9ebea183-177f-464c-b256-9530837ccb86/results-00000-of-00001, err: 503
ERROR apache_beam.runners.common:common.py:1546 Encountered exceptions in finalize_write: [503] [while running 'WriteToText/Write/WriteImpl/FinalizeWrite']
Traceback (most recent call last):
File "<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/runners/common.py",> line 1435, in process
return self.do_fn_invoker.invoke_process(windowed_value)
File "<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/runners/common.py",> line 851, in invoke_process
self._invoke_process_per_window(
File "<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/runners/common.py",> line 995, in _invoke_process_per_window
self.output_handler.handle_process_outputs(
File "<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/runners/common.py",> line 1611, in handle_process_outputs
for result in results:
File "<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/iobase.py",> line 1248, in <genexpr>
return (
File "<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/filebasedsink.py",> line 352, in finalize_write
raise Exception(
Exception: Encountered exceptions in finalize_write: [503]
INFO apache_beam.io.gcp.gcsio:gcsio.py:491 Finished listing 1 files in 0.05826902389526367 seconds.
=============================== warnings summary ===============================
apache_beam/io/gcp/bigquery.py:2603
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/bigquery.py>:2603: DeprecationWarning: invalid escape sequence \#
"""Read data from BigQuery.
apache_beam/examples/complete/game/hourly_team_score_it_test.py::HourlyTeamScoreIT::test_hourly_team_score_output_checksum_on_small_input
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/tests/utils.py>:63: PendingDeprecationWarning: Client.dataset is deprecated and will be removed in a future version. Use a string like 'my_project.my_dataset' or a cloud.google.bigquery.DatasetReference object, instead.
dataset_ref = client.dataset(unique_dataset_name, project=project)
apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py::BigqueryTornadoesIT::test_bigquery_tornadoes_it
apache_beam/examples/cookbook/filters_test.py::FiltersTest::test_filters_output_bigquery_matcher
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/io/gcp/tests/utils.py>:100: PendingDeprecationWarning: Client.dataset is deprecated and will be removed in a future version. Use a string like 'my_project.my_dataset' or a cloud.google.bigquery.DatasetReference object, instead.
table_ref = client.dataset(dataset_id).table(table_id)
apache_beam/examples/dataframe/flight_delays_it_test.py: 46 warnings
<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/apache_beam/examples/dataframe/flight_delays.py>:47: FutureWarning: The default value of numeric_only in DataFrame.mean is deprecated. In a future version, it will default to False. In addition, specifying 'numeric_only=None' is deprecated. Select only valid columns or specify the value of numeric_only to silence this warning.
return airline_df[at_top_airports].mean()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
- generated xml file: <https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/pytest_postCommitExamples-direct-py39.xml> -
=========================== short test summary info ============================
FAILED apache_beam/examples/cookbook/bigquery_side_input_it_test.py::BigQuerySideInputIT::test_bigquery_side_input_it - Exception: Encountered exceptions in finalize_write: [503] [while running 'WriteToText/Write/WriteImpl/FinalizeWrite']
= 1 failed, 22 passed, 18 skipped, 7255 deselected, 50 warnings in 235.21s (0:03:55) =
> Task :sdks:python:test-suites:direct:py39:examples FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/ws/src/sdks/python/test-suites/direct/common.gradle'> line: 93
* What went wrong:
Execution failed for task ':sdks:python:test-suites:direct:py39:examples'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.4/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 10m 15s
18 actionable tasks: 17 executed, 1 from cache
Publishing build scan...
The request was rejected.
Gradle Enterprise plugin version 3.16 is newer than the newest version supported by Gradle Enterprise 2023.3.4 which is 3.15. Please update to a newer version of Gradle Enterprise.
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal : beam_PostCommit_Python_Examples_Direct #2751
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_Python_Examples_Direct/2751/display/redirect?page=changes>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org