You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@beam.apache.org by "Juta Staes (JIRA)" <ji...@apache.org> on 2019/02/25 15:11:00 UTC

[jira] [Comment Edited] (BEAM-6711) Bigquery Tornadoes IT is broken in Python3 PostCommit test suite.

    [ https://issues.apache.org/jira/browse/BEAM-6711?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16776981#comment-16776981 ] 

Juta Staes edited comment on BEAM-6711 at 2/25/19 3:10 PM:
-----------------------------------------------------------

I still get an error in another test:
{noformat}
ERROR: test_records_traverse_transform_with_mocks (apache_beam.io.gcp.bigquery_file_loads_test.TestBigQueryFileLoads)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py", line 311, in test_records_traverse_transform_with_mocks
    equal_to([job_reference]), label='CheckJobs')
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/pipeline.py", line 426, in __exit__
    self.run().wait_until_finish()
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/testing/test_pipeline.py", line 107, in run
    else test_runner_api))
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/pipeline.py", line 406, in run
    self._options).run(False)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/pipeline.py", line 419, in run
    return self.runner.run_pipeline(self, self._options)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/direct/direct_runner.py", line 132, in run_pipeline
    return runner.run_pipeline(pipeline, options)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 276, in run_pipeline
    default_environment=self._default_environment))
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 280, in run_via_runner_api
    return self.run_stages(*self.create_stages(pipeline_proto))
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 356, in run_stages
    stage_context.safe_coders)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 511, in run_stage
    data_input, data_output)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 1215, in process_bundle
    result_future = self._controller.control_handler.push(process_bundle)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 830, in push
    response = self.worker.do_instruction(request)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/sdk_worker.py", line 265, in do_instruction
    request.instruction_id)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/sdk_worker.py", line 281, in process_bundle
    delayed_applications = bundle_processor.process_bundle(instruction_id)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/bundle_processor.py", line 549, in process_bundle
    ].process_encoded(data.data)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/bundle_processor.py", line 140, in process_encoded
    self.output(decoded_value)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/operations.py", line 223, in output
    cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/operations.py", line 98, in receive
    cython.cast(Operation, consumer).process(windowed_value)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/operations.py", line 725, in process
    entry = self.table.get(wkey, None)
TypeError: unhashable type: 'TableReference'
{noformat}


was (Author: juta):
I still get an error in another test suite:
{noformat}
ERROR: test_records_traverse_transform_with_mocks (apache_beam.io.gcp.bigquery_file_loads_test.TestBigQueryFileLoads)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/io/gcp/bigquery_file_loads_test.py", line 311, in test_records_traverse_transform_with_mocks
    equal_to([job_reference]), label='CheckJobs')
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/pipeline.py", line 426, in __exit__
    self.run().wait_until_finish()
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/testing/test_pipeline.py", line 107, in run
    else test_runner_api))
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/pipeline.py", line 406, in run
    self._options).run(False)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/pipeline.py", line 419, in run
    return self.runner.run_pipeline(self, self._options)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/direct/direct_runner.py", line 132, in run_pipeline
    return runner.run_pipeline(pipeline, options)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 276, in run_pipeline
    default_environment=self._default_environment))
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 280, in run_via_runner_api
    return self.run_stages(*self.create_stages(pipeline_proto))
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 356, in run_stages
    stage_context.safe_coders)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 511, in run_stage
    data_input, data_output)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 1215, in process_bundle
    result_future = self._controller.control_handler.push(process_bundle)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/portability/fn_api_runner.py", line 830, in push
    response = self.worker.do_instruction(request)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/sdk_worker.py", line 265, in do_instruction
    request.instruction_id)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/sdk_worker.py", line 281, in process_bundle
    delayed_applications = bundle_processor.process_bundle(instruction_id)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/bundle_processor.py", line 549, in process_bundle
    ].process_encoded(data.data)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/bundle_processor.py", line 140, in process_encoded
    self.output(decoded_value)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/operations.py", line 223, in output
    cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/operations.py", line 98, in receive
    cython.cast(Operation, consumer).process(windowed_value)
  File "/mnt/c/Users/Juta/Documents/02-projects/apache/beam/sdks/python/apache_beam/runners/worker/operations.py", line 725, in process
    entry = self.table.get(wkey, None)
TypeError: unhashable type: 'TableReference'
{noformat}

> Bigquery Tornadoes IT is broken in Python3 PostCommit test suite. 
> ------------------------------------------------------------------
>
>                 Key: BEAM-6711
>                 URL: https://issues.apache.org/jira/browse/BEAM-6711
>             Project: Beam
>          Issue Type: Sub-task
>          Components: sdk-py-core
>            Reporter: Valentyn Tymofieiev
>            Assignee: Pablo Estrada
>            Priority: Major
>          Time Spent: 6h 50m
>  Remaining Estimate: 0h
>
> First failure was observed in https://builds.apache.org/job/beam_PostCommit_Python3_Verify/54 , after 
> https://github.com/apache/beam/commit/cdea885872b3be7de9ba22f22700be89f7d53766 was merged. 
> [~pabloem], could you please take a look? I suggest we do a rollback + rollforward with a fix.
> {noformat}
> root: ERROR: Exception at bundle <apache_beam.runners.direct.bundle_factory._Bundle object at 0x7f35e5331240>, due to an exception.
>  Traceback (most recent call last):
>   File "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python3_Verify/src/sdks/python/apache_beam/runners/common.py", line 727, in process
>     return self.do_fn_invoker.invoke_process(windowed_value)
>   File "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python3_Verify/src/sdks/python/apache_beam/runners/common.py", line 556, in invoke_process
>     windowed_value, additional_args, additional_kwargs, output_processor)
>   File "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python3_Verify/src/sdks/python/apache_beam/runners/common.py", line 622, in _invoke_per_window
>     self.process_method(*args_for_process, **kwargs_for_process))
>   File "/home/jenkins/jenkins-slave/workspace/beam_PostCommit_Python3_Verify/src/sdks/python/apache_beam/runners/common.py", line 823, in process_outputs
>     for result in results:
>   File "/home/jenkins/jenkins-slave/works
> pace/beam_PostCommit_Python3_Verify/src/sdks/python/apache_beam/io/gcp/bigquery_file_loads.py", line 191, in process
>     if destination in self._destination_to_file_writer:
> TypeError: unhashable type: 'TableReference'
> {noformat}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)