You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2019/01/25 19:19:07 UTC
Build failed in Jenkins: beam_PostCommit_Python_Verify #7195
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/7195/display/redirect>
------------------------------------------
[...truncated 283.05 KB...]
###########################################################################
# Build pipeline options if not provided in --pipeline_opts from commandline
if [[ -z $PIPELINE_OPTS ]]; then
# Check that the script is running in a known directory.
if [[ $PWD != *sdks/python* ]]; then
echo 'Unable to locate Apache Beam Python SDK root directory'
exit 1
fi
# Go to the Apache Beam Python SDK root
if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
# Create a tarball if not exists
if [[ $(find ${SDK_LOCATION}) ]]; then
SDK_LOCATION=$(find ${SDK_LOCATION})
else
python setup.py -q sdist
SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
fi
# Install test dependencies for ValidatesRunner tests.
echo "pyhamcrest" > postcommit_requirements.txt
echo "mock" >> postcommit_requirements.txt
# Options used to run testing pipeline on Cloud Dataflow Service. Also used for
# running on DirectRunner (some options ignored).
opts=(
"--runner=$RUNNER"
"--project=$PROJECT"
"--staging_location=$GCS_LOCATION/staging-it"
"--temp_location=$GCS_LOCATION/temp-it"
"--output=$GCS_LOCATION/py-it-cloud/output"
"--sdk_location=$SDK_LOCATION"
"--requirements_file=postcommit_requirements.txt"
"--num_workers=$NUM_WORKERS"
"--sleep_secs=$SLEEP_SECS"
)
# Add --streaming if provided
if [[ "$STREAMING" = true ]]; then
opts+=("--streaming")
fi
# Add --dataflow_worker_jar if provided
if [[ ! -z "$WORKER_JAR" ]]; then
opts+=("--dataflow_worker_jar=$WORKER_JAR")
fi
if [[ ! -z "$KMS_KEY_NAME" ]]; then
opts+=("--kms_key_name=$KMS_KEY_NAME")
fi
PIPELINE_OPTS=$(IFS=" " ; echo "${opts[*]}")
fi
pwd | sed 's/sdks\/python.*/sdks\/python/'
find ${SDK_LOCATION}
find ${SDK_LOCATION}
IFS=" " ; echo "${opts[*]}"
###########################################################################
# Run tests and validate that jobs finish successfully.
echo ">>> RUNNING integration tests with pipeline options: $PIPELINE_OPTS"
>>> RUNNING integration tests with pipeline options: --runner=TestDataflowRunner --project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --sdk_location=build/apache-beam.tar.gz --requirements_file=postcommit_requirements.txt --num_workers=1 --sleep_secs=20 --kms_key_name=projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test/cryptoKeyVersions/1
python setup.py nosetests \
--test-pipeline-options="$PIPELINE_OPTS" \
$TEST_OPTS
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/build/gradleenv/1327086738/local/lib/python2.7/site-packages/setuptools/dist.py>:470: UserWarning: Normalizing '2.11.0.dev' to '2.11.0.dev0'
normalized_version,
running nosetests
running egg_info
writing requirements to apache_beam.egg-info/requires.txt
writing apache_beam.egg-info/PKG-INFO
writing top-level names to apache_beam.egg-info/top_level.txt
writing dependency_links to apache_beam.egg-info/dependency_links.txt
writing entry points to apache_beam.egg-info/entry_points.txt
reading manifest file 'apache_beam.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
writing manifest file 'apache_beam.egg-info/SOURCES.txt'
test_bigquery_tornadoes_it (apache_beam.examples.cookbook.bigquery_tornadoes_it_test.BigqueryTornadoesIT) ... ok
test_autocomplete_it (apache_beam.examples.complete.autocomplete_test.AutocompleteTest) ... ok
test_datastore_wordcount_it (apache_beam.examples.cookbook.datastore_wordcount_it_test.DatastoreWordCountIT) ... ok
test_leader_board_it (apache_beam.examples.complete.game.leader_board_it_test.LeaderBoardIT) ... ok
test_streaming_wordcount_it (apache_beam.examples.streaming_wordcount_it_test.StreamingWordCountIT) ... ok
test_wordcount_fnapi_it (apache_beam.examples.wordcount_it_test.WordCountIT) ... ok
test_wordcount_it (apache_beam.examples.wordcount_it_test.WordCountIT) ... ok
test_game_stats_it (apache_beam.examples.complete.game.game_stats_it_test.GameStatsIT) ... ok
test_copy (apache_beam.io.gcp.gcsio_integration_test.GcsIOIntegrationTest) ... ok
test_copy_batch (apache_beam.io.gcp.gcsio_integration_test.GcsIOIntegrationTest) ... ok
test_copy_batch_kms (apache_beam.io.gcp.gcsio_integration_test.GcsIOIntegrationTest) ... ok
test_copy_batch_rewrite_token (apache_beam.io.gcp.gcsio_integration_test.GcsIOIntegrationTest) ... ok
test_copy_kms (apache_beam.io.gcp.gcsio_integration_test.GcsIOIntegrationTest) ... ok
test_copy_rewrite_token (apache_beam.io.gcp.gcsio_integration_test.GcsIOIntegrationTest) ... ok
test_avro_it (apache_beam.examples.fastavro_it_test.FastavroIT) ... ok
test_user_score_it (apache_beam.examples.complete.game.user_score_it_test.UserScoreIT) ... ok
test_hourly_team_score_it (apache_beam.examples.complete.game.hourly_team_score_it_test.HourlyTeamScoreIT) ... ok
test_bigquery_read_1M_python (apache_beam.io.gcp.bigquery_io_read_it_test.BigqueryIOReadIT) ... ok
test_parquetio_it (apache_beam.io.parquetio_it_test.TestParquetIT) ... ok
test_streaming_data_only (apache_beam.io.gcp.pubsub_integration_test.PubSubIntegrationTest) ... ok
test_streaming_with_attributes (apache_beam.io.gcp.pubsub_integration_test.PubSubIntegrationTest) ... ok
test_big_query_legacy_sql (apache_beam.io.gcp.big_query_query_to_table_it_test.BigQueryQueryToTableIT) ... ok
test_big_query_new_types (apache_beam.io.gcp.big_query_query_to_table_it_test.BigQueryQueryToTableIT) ... ok
test_big_query_standard_sql (apache_beam.io.gcp.big_query_query_to_table_it_test.BigQueryQueryToTableIT) ... ok
test_datastore_write_limit (apache_beam.io.gcp.datastore_write_it_test.DatastoreWriteIT) ... ERROR
======================================================================
ERROR: test_datastore_write_limit (apache_beam.io.gcp.datastore_write_it_test.DatastoreWriteIT)
----------------------------------------------------------------------
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py",> line 65, in test_datastore_write_limit
self.run_datastore_write(limit=self.LIMIT)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py",> line 61, in run_datastore_write
**extra_opts))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/gcp/datastore_write_it_pipeline.py",> line 202, in run
p.run()
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/testing/test_pipeline.py",> line 107, in run
else test_runner_api))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/pipeline.py",> line 405, in run
self._options).run(False)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/pipeline.py",> line 418, in run
return self.runner.run_pipeline(self, self._options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py",> line 53, in run_pipeline
pipeline, options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py",> line 394, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/utils/retry.py",> line 184, in wrapper
return fun(*args, **kwargs)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",> line 504, in create_job
self.create_job_description(job)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",> line 533, in create_job_description
resources = self._stage_resources(job.options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py",> line 464, in _stage_resources
staging_location=google_cloud_options.staging_location)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/stager.py",> line 167, in stage_job_resources
requirements_cache_path)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/utils/retry.py",> line 184, in wrapper
return fun(*args, **kwargs)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/stager.py",> line 427, in _populate_requirements_cache
processes.check_output(cmd_args, stderr=processes.STDOUT)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/utils/processes.py",> line 53, in check_output
return subprocess.check_output(*args, **kwargs)
File "/usr/lib/python2.7/subprocess.py", line 567, in check_output
process = Popen(stdout=PIPE, *popenargs, **kwargs)
File "/usr/lib/python2.7/subprocess.py", line 711, in __init__
errread, errwrite)
File "/usr/lib/python2.7/subprocess.py", line 1235, in _execute_child
self.pid = os.fork()
OSError: [Errno 11] Resource temporarily unavailable
-------------------- >> begin captured logging << --------------------
root: INFO: Writing 1001 entities to apache-beam-testing
root: WARNING: datastoreio write transform is experimental.
root: DEBUG: Connecting using Google Application Default Credentials.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/pipeline.pb...
oauth2client.transport: INFO: Attempting refresh to obtain initial access_token
oauth2client.transport: INFO: Attempting refresh to obtain initial access_token
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/pipeline.pb in 0 seconds.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/requirements.txt...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/requirements.txt in 0 seconds.
root: INFO: Executing command: ['<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/build/gradleenv/1327086738/bin/python',> '-m', 'pip', 'download', '--dest', '/tmp/dataflow-requirements-cache', '-r', 'postcommit_requirements.txt', '--exists-action', 'i', '--no-binary', ':all:']
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/PyHamcrest-1.9.0.tar.gz...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/PyHamcrest-1.9.0.tar.gz in 0 seconds.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/pbr-5.1.1.tar.gz...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/pbr-5.1.1.tar.gz in 0 seconds.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/mock-2.0.0.tar.gz...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/mock-2.0.0.tar.gz in 0 seconds.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/six-1.12.0.tar.gz...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/six-1.12.0.tar.gz in 0 seconds.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/setuptools-40.6.3.zip...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/setuptools-40.6.3.zip in 0 seconds.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/funcsigs-1.0.2.tar.gz...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/funcsigs-1.0.2.tar.gz in 0 seconds.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/pickled_main_session...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/pickled_main_session in 0 seconds.
root: INFO: Copying Beam SDK "build/apache-beam.tar.gz" to staging location.
root: INFO: Starting GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/dataflow_python_sdk.tar...
root: INFO: Completed GCS upload to gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999/dataflow_python_sdk.tar in 0 seconds.
root: INFO: Parsing unknown args: ['--output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output', '--sleep_secs=20', '--kms_key_name=projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test/cryptoKeyVersions/1']
root: INFO: Parsing unknown args: ['--output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output', '--sleep_secs=20', '--kms_key_name=projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test/cryptoKeyVersions/1']
root: DEBUG: JOB: {
"environment": {
"clusterManagerApiService": "compute.googleapis.com",
"dataset": "bigquery.googleapis.com/cloud_dataflow",
"sdkPipelineOptions": {
"display_data": [
{
"key": "requirements_file",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "STRING",
"value": "postcommit_requirements.txt"
},
{
"key": "sleep_secs",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "STRING",
"value": "20"
},
{
"key": "sdk_location",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "STRING",
"value": "build/apache-beam.tar.gz"
},
{
"key": "save_main_session",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "BOOLEAN",
"value": true
},
{
"key": "num_workers",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "INTEGER",
"value": 1
},
{
"key": "runner",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "STRING",
"value": "TestDataflowRunner"
},
{
"key": "staging_location",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "STRING",
"value": "gs://temp-storage-for-end-to-end-tests/staging-it/beamapp-jenkins-0125185308-715877.1548442388.715999"
},
{
"key": "project",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "STRING",
"value": "apache-beam-testing"
},
{
"key": "kms_key_name",
"namespace": "apache_beam.options.pipeline_options.PipelineOptions",
"type": "STRING",
"value": "projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test/cryptoKeyVersions/1"
},
{
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal : beam_PostCommit_Python_Verify
#7197
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/7197/display/redirect?page=changes>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_Python_Verify #7196
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/7196/display/redirect?page=changes>
Changes:
[scott] Add additional debug output for metrics prober test failures
[scott] Add a bit more details to the CONTRIBUTING stub
------------------------------------------
[...truncated 115.12 KB...]
Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.11.0.dev0)
Collecting requests>=2.7.0 (from hdfs<3.0.0,>=2.1.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/7d/e3/20f3d364d6c8e5d2353c72a67778eb189176f08e873c9900e10c0287b84b/requests-2.21.0-py2.py3-none-any.whl
Collecting pbr>=0.11 (from mock<3.0.0,>=1.0.1->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/f3/04/fddc1c2dd75b256eda4d360024692231a2c19a0c61ad7f4a162407c1ab58/pbr-5.1.1-py2.py3-none-any.whl
Collecting funcsigs>=1; python_version < "3.3" (from mock<3.0.0,>=1.0.1->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/69/cb/f5be453359271714c01b9bd06126eaf2e368f1fddfff30818754b5ac2328/funcsigs-1.0.2-py2.py3-none-any.whl
Collecting pyasn1>=0.1.7 (from oauth2client<4,>=2.0.1->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/7b/7c/c9386b82a25115cccf1903441bba3cbadcfae7b678a20167347fa8ded34c/pyasn1-0.4.5-py2.py3-none-any.whl
Collecting pyasn1-modules>=0.0.5 (from oauth2client<4,>=2.0.1->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/59/84/339c2de112d7448d7d4c500b8016783d9eac8981f89ec1d58fdd4685d97f/pyasn1_modules-0.2.3-py2.py3-none-any.whl
Collecting rsa>=3.1.4 (from oauth2client<4,>=2.0.1->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/02/e5/38518af393f7c214357079ce67a317307936896e961e35450b70fad2a9cf/rsa-4.0-py2.py3-none-any.whl
Requirement already satisfied: setuptools in <https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/build/gradleenv/1327086738/lib/python2.7/site-packages> (from protobuf<4,>=3.5.0.post1->apache-beam==2.11.0.dev0) (40.6.3)
Collecting pyparsing>=2.1.4 (from pydot<1.3,>=1.2.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/de/0a/001be530836743d8be6c2d85069f46fecf84ac6c18c7f5fb8125ee11d854/pyparsing-2.3.1-py2.py3-none-any.whl
Collecting fasteners>=0.14 (from google-apitools<0.5.27,>=0.5.26->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/14/3a/096c7ad18e102d4f219f5dd15951f9728ca5092a3385d2e8f79a7c1e1017/fasteners-0.14.1-py2.py3-none-any.whl
Collecting googleapis-common-protos<2.0dev,>=1.5.2 (from proto-google-cloud-datastore-v1<=0.90.4,>=0.90.0->apache-beam==2.11.0.dev0)
Collecting google-api-core[grpc]<2.0.0dev,>=1.4.1 (from google-cloud-pubsub==0.39.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/8b/01/13758ff9b970008ccf9e0dcc3b86d0e01937d7485b9a2c6142c9c2bdb4da/google_api_core-1.7.0-py2.py3-none-any.whl
Collecting grpc-google-iam-v1<0.12dev,>=0.11.4 (from google-cloud-pubsub==0.39.0->apache-beam==2.11.0.dev0)
Collecting google-cloud-core<0.30dev,>=0.28.0 (from google-cloud-bigquery<1.7.0,>=1.6.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/0c/f2/3c225e7a69cb27d283b68bff867722bd066bc1858611180197f711815ea5/google_cloud_core-0.29.1-py2.py3-none-any.whl
Collecting google-resumable-media>=0.2.1 (from google-cloud-bigquery<1.7.0,>=1.6.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/e2/5d/4bc5c28c252a62efe69ed1a1561da92bd5af8eca0cdcdf8e60354fae9b29/google_resumable_media-0.3.2-py2.py3-none-any.whl
Collecting python-dateutil>=2.5.0 (from pandas<0.24,>=0.23.4->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/74/68/d87d9b36af36f44254a8d512cbfc48369103a3b9e474be9bdfe536abfc45/python_dateutil-2.7.5-py2.py3-none-any.whl
Collecting monotonic>=0.6; python_version == "2.7" (from tenacity<6.0,>=5.0.2->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/ac/aa/063eca6a416f397bd99552c534c6d11d57f58f2e94c14780f3bbf818c4cf/monotonic-1.5-py2.py3-none-any.whl
Collecting certifi>=2017.4.17 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/9f/e0/accfc1b56b57e9750eba272e24c4dddeac86852c2bebd1236674d7887e8a/certifi-2018.11.29-py2.py3-none-any.whl
Collecting chardet<3.1.0,>=3.0.2 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl
Collecting idna<2.9,>=2.5 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/14/2c/cd551d81dbe15200be1cf41cd03869a46fe7226e7450af7a6545bfc474c9/idna-2.8-py2.py3-none-any.whl
Collecting urllib3<1.25,>=1.21.1 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/62/00/ee1d7de624db8ba7090d1226aebefab96a2c71cd5cfa7629d6ad3f61b79e/urllib3-1.24.1-py2.py3-none-any.whl
Collecting google-auth<2.0dev,>=0.4.0 (from google-api-core[grpc]<2.0.0dev,>=1.4.1->google-cloud-pubsub==0.39.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/4e/85/71b2dfbf5b4241cd031cc333ed71f90a271074a97cb2c517bb65f07a1a90/google_auth-1.6.2-py2.py3-none-any.whl
Collecting cachetools>=2.0.0 (from google-auth<2.0dev,>=0.4.0->google-api-core[grpc]<2.0.0dev,>=1.4.1->google-cloud-pubsub==0.39.0->apache-beam==2.11.0.dev0)
Using cached https://files.pythonhosted.org/packages/76/7e/08cd3846bebeabb6b1cfc4af8aae649d90249b4aeed080bddb5297f1d73b/cachetools-3.0.0-py2.py3-none-any.whl
Installing collected packages: crcmod, dill, fastavro, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, funcsigs, mock, pyasn1, pyasn1-modules, rsa, oauth2client, pyparsing, pydot, pytz, pyvcf, pyyaml, avro, typing, numpy, pyarrow, monotonic, fasteners, google-apitools, googleapis-common-protos, proto-google-cloud-datastore-v1, cachetools, google-auth, google-api-core, grpc-google-iam-v1, google-cloud-pubsub, google-cloud-core, google-resumable-media, google-cloud-bigquery, googledatastore, nose, python-dateutil, pandas, parameterized, pyhamcrest, tenacity, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-1.8.2 cachetools-3.0.0 certifi-2018.11.29 chardet-3.0.4 crcmod-1.7 dill-0.2.9 docopt-0.6.2 fastavro-0.21.17 fasteners-0.14.1 funcsigs-1.0.2 google-api-core-1.7.0 google-apitools-0.5.26 google-auth-1.6.2 google-cloud-bigquery-1.6.1 google-cloud-core-0.29.1 google-cloud-pubsub-0.39.0 google-resumable-media-0.3.2 googleapis-common-protos-1.5.6 googledatastore-7.0.2 grpc-google-iam-v1-0.11.4 hdfs-2.2.2 httplib2-0.11.3 idna-2.8 mock-2.0.0 monotonic-1.5 nose-1.3.7 numpy-1.16.0 oauth2client-3.0.0 pandas-0.23.4 parameterized-0.6.3 pbr-5.1.1 proto-google-cloud-datastore-v1-0.90.4 pyarrow-0.11.1 pyasn1-0.4.5 pyasn1-modules-0.2.3 pydot-1.2.4 pyhamcrest-1.9.0 pyparsing-2.3.1 python-dateutil-2.7.5 pytz-2018.9 pyvcf-0.6.8 pyyaml-3.13 requests-2.21.0 rsa-4.0 tenacity-5.0.2 typing-3.6.6 urllib3-1.24.1
> Task :beam-sdks-python:directRunnerIT
###########################################################################
# Build pipeline options if not provided in --pipeline_opts from commandline
if [[ -z $PIPELINE_OPTS ]]; then
# Check that the script is running in a known directory.
if [[ $PWD != *sdks/python* ]]; then
echo 'Unable to locate Apache Beam Python SDK root directory'
exit 1
fi
# Go to the Apache Beam Python SDK root
if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
# Create a tarball if not exists
if [[ $(find ${SDK_LOCATION}) ]]; then
SDK_LOCATION=$(find ${SDK_LOCATION})
else
python setup.py -q sdist
SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
fi
# Install test dependencies for ValidatesRunner tests.
echo "pyhamcrest" > postcommit_requirements.txt
echo "mock" >> postcommit_requirements.txt
# Options used to run testing pipeline on Cloud Dataflow Service. Also used for
# running on DirectRunner (some options ignored).
opts=(
"--runner=$RUNNER"
"--project=$PROJECT"
"--staging_location=$GCS_LOCATION/staging-it"
"--temp_location=$GCS_LOCATION/temp-it"
"--output=$GCS_LOCATION/py-it-cloud/output"
"--sdk_location=$SDK_LOCATION"
"--requirements_file=postcommit_requirements.txt"
"--num_workers=$NUM_WORKERS"
"--sleep_secs=$SLEEP_SECS"
)
# Add --streaming if provided
if [[ "$STREAMING" = true ]]; then
opts+=("--streaming")
fi
# Add --dataflow_worker_jar if provided
if [[ ! -z "$WORKER_JAR" ]]; then
opts+=("--dataflow_worker_jar=$WORKER_JAR")
fi
if [[ ! -z "$KMS_KEY_NAME" ]]; then
opts+=("--kms_key_name=$KMS_KEY_NAME")
fi
PIPELINE_OPTS=$(IFS=" " ; echo "${opts[*]}")
fi
pwd | sed 's/sdks\/python.*/sdks\/python/'
find ${SDK_LOCATION}
find ${SDK_LOCATION}
IFS=" " ; echo "${opts[*]}"
###########################################################################
# Run tests and validate that jobs finish successfully.
>>> RUNNING integration tests with pipeline options: --runner=TestDirectRunner --project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --sdk_location=build/apache-beam.tar.gz --requirements_file=postcommit_requirements.txt --num_workers=1 --sleep_secs=20 --kms_key_name=projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test/cryptoKeyVersions/1
echo ">>> RUNNING integration tests with pipeline options: $PIPELINE_OPTS"
python setup.py nosetests \
--test-pipeline-options="$PIPELINE_OPTS" \
$TEST_OPTS
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/build/gradleenv/1327086738/local/lib/python2.7/site-packages/setuptools/dist.py>:470: UserWarning: Normalizing '2.11.0.dev' to '2.11.0.dev0'
normalized_version,
running nosetests
running egg_info
writing requirements to apache_beam.egg-info/requires.txt
writing apache_beam.egg-info/PKG-INFO
writing top-level names to apache_beam.egg-info/top_level.txt
writing dependency_links to apache_beam.egg-info/dependency_links.txt
writing entry points to apache_beam.egg-info/entry_points.txt
reading manifest file 'apache_beam.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
writing manifest file 'apache_beam.egg-info/SOURCES.txt'
Traceback (most recent call last):
File "src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi", line 33, in grpc._cython.cygrpc._spawn_callback_async
File "src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi", line 24, in grpc._cython.cygrpc._spawn_callback_in_thread
File "src/python/grpcio/grpc/_cython/_cygrpc/fork_posix.pyx.pxi", line 107, in grpc._cython.cygrpc.ForkManagedThread.start
File "/usr/lib/python2.7/threading.py", line 736, in start
_start_new_thread(self.__bootstrap, ())
thread.error: can't start new thread
Exception thread.error: error("can't start new thread",) in 'grpc._cython.cygrpc._get_metadata' ignored
test_bigquery_read_1M_python (apache_beam.io.gcp.bigquery_io_read_it_test.BigqueryIOReadIT) ... ERROR
test_wordcount_it (apache_beam.examples.wordcount_it_test.WordCountIT) ... ERROR
test_big_query_legacy_sql (apache_beam.io.gcp.big_query_query_to_table_it_test.BigQueryQueryToTableIT) ... ERROR
test_big_query_new_types (apache_beam.io.gcp.big_query_query_to_table_it_test.BigQueryQueryToTableIT) ... ERROR
test_big_query_standard_sql (apache_beam.io.gcp.big_query_query_to_table_it_test.BigQueryQueryToTableIT) ... ERROR
test_streaming_data_only (apache_beam.io.gcp.pubsub_integration_test.PubSubIntegrationTest) ... ok
test_streaming_with_attributes (apache_beam.io.gcp.pubsub_integration_test.PubSubIntegrationTest) ... ok
======================================================================
ERROR: test_bigquery_read_1M_python (apache_beam.io.gcp.bigquery_io_read_it_test.BigqueryIOReadIT)
----------------------------------------------------------------------
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/gcp/bigquery_io_read_it_test.py",> line 58, in test_bigquery_read_1M_python
self.run_bigquery_io_read_pipeline('1M')
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/gcp/bigquery_io_read_it_test.py",> line 54, in run_bigquery_io_read_pipeline
**extra_opts))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/io/gcp/bigquery_io_read_pipeline.py",> line 74, in run
p.run()
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/testing/test_pipeline.py",> line 107, in run
else test_runner_api))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/pipeline.py",> line 405, in run
self._options).run(False)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/pipeline.py",> line 418, in run
return self.runner.run_pipeline(self, self._options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/test_direct_runner.py",> line 43, in run_pipeline
self.result = super(TestDirectRunner, self).run_pipeline(pipeline, options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/direct_runner.py",> line 136, in run_pipeline
return runner.run_pipeline(pipeline, options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/direct_runner.py",> line 398, in run_pipeline
evaluation_context)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/executor.py",> line 394, in __init__
self._executor = _ExecutorServiceParallelExecutor(*args, **kwargs)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/executor.py",> line 414, in __init__
_ExecutorServiceParallelExecutor.NUM_WORKERS)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/executor.py",> line 108, in __init__
self.queue, i) for i in range(num_workers)]
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/executor.py",> line 69, in __init__
self.start()
File "/usr/lib/python2.7/threading.py", line 736, in start
_start_new_thread(self.__bootstrap, ())
error: can't start new thread
-------------------- >> begin captured logging << --------------------
root: INFO: Running pipeline with DirectRunner.
--------------------- >> end captured logging << ---------------------
======================================================================
ERROR: test_wordcount_it (apache_beam.examples.wordcount_it_test.WordCountIT)
----------------------------------------------------------------------
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/examples/wordcount_it_test.py",> line 47, in test_wordcount_it
self._run_wordcount_it()
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/examples/wordcount_it_test.py",> line 75, in _run_wordcount_it
wordcount.run(test_pipeline.get_full_options_as_args(**extra_opts))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/examples/wordcount.py",> line 115, in run
result = p.run()
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/pipeline.py",> line 405, in run
self._options).run(False)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/pipeline.py",> line 418, in run
return self.runner.run_pipeline(self, self._options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/test_direct_runner.py",> line 43, in run_pipeline
self.result = super(TestDirectRunner, self).run_pipeline(pipeline, options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/direct/direct_runner.py",> line 136, in run_pipeline
return runner.run_pipeline(pipeline, options)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/fn_api_runner.py",> line 274, in run_pipeline
default_environment=self._default_environment))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/fn_api_runner.py",> line 277, in run_via_runner_api
return self.run_stages(*self.create_stages(pipeline_proto))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/fn_api_runner.py",> line 364, in run_stages
safe_coders)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/fn_api_runner.py",> line 510, in run_stage
self._progress_frequency).process_bundle(data_input, data_output)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/fn_api_runner.py",> line 1107, in process_bundle
result_future = self._controller.control_handler.push(process_bundle)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/portability/fn_api_runner.py",> line 760, in push
response = self.worker.do_instruction(request)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/sdk_worker.py",> line 259, in do_instruction
request.instruction_id)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/sdk_worker.py",> line 275, in process_bundle
bundle_processor.process_bundle(instruction_id)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/bundle_processor.py",> line 496, in process_bundle
].process_encoded(data.data)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/bundle_processor.py",> line 126, in process_encoded
self.output(decoded_value)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/operations.py",> line 183, in output
cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/operations.py",> line 89, in receive
cython.cast(Operation, consumer).process(windowed_value)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/operations.py",> line 498, in process
self.dofn_receiver.receive(o)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/common.py",> line 678, in receive
self.process(windowed_value)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/common.py",> line 684, in process
self._reraise_augmented(exn)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/common.py",> line 682, in process
self.do_fn_invoker.invoke_process(windowed_value)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/common.py",> line 419, in invoke_process
windowed_value, self.process_method(windowed_value.value))
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/common.py",> line 792, in process_outputs
self.main_receivers.receive(windowed_value)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/operations.py",> line 89, in receive
cython.cast(Operation, consumer).process(windowed_value)
File "<https://builds.apache.org/job/beam_PostCommit_Python_Verify/ws/src/sdks/python/apache_beam/runners/worker/operations.py",> line 498, in process
self.dofn_receiver.receive(o)
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org